From e423784fd0f4e057f3f58de226600066008f263c Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Wed, 21 Feb 2018 23:26:24 +0900 Subject: [PATCH 01/24] Added specialized SegmentPostings when there are no DeleteSet --- src/core/inverted_index_reader.rs | 25 ++++-- src/core/segment_reader.rs | 36 +++++---- src/fastfield/delete.rs | 30 ++++--- src/indexer/merger.rs | 46 ++++++----- src/postings/delete_set.rs | 27 +++++++ src/postings/mod.rs | 89 ++++++++++----------- src/postings/segment_postings.rs | 67 ++++++++-------- src/query/boolean_query/boolean_weight.rs | 95 +++++++++++++++++------ src/query/boolean_query/mod.rs | 6 +- src/query/intersection_two.rs | 73 +++++++++++++++++ src/query/mod.rs | 2 + src/query/phrase_query/phrase_scorer.rs | 7 +- src/query/term_query/term_scorer.rs | 13 ++-- src/query/term_query/term_weight.rs | 68 ++++++++++------ 14 files changed, 401 insertions(+), 183 deletions(-) create mode 100644 src/postings/delete_set.rs create mode 100644 src/query/intersection_two.rs diff --git a/src/core/inverted_index_reader.rs b/src/core/inverted_index_reader.rs index cea7e4e4b..3eb11165d 100644 --- a/src/core/inverted_index_reader.rs +++ b/src/core/inverted_index_reader.rs @@ -7,6 +7,7 @@ use schema::Term; use fastfield::DeleteBitSet; use compression::CompressedIntStream; use postings::FreqReadingOption; +use postings::{DeleteSet, NoDelete}; /// The inverted index reader is in charge of accessing /// the inverted index associated to a specific field. @@ -25,7 +26,7 @@ pub struct InvertedIndexReader { termdict: TermDictionaryImpl, postings_source: ReadOnlySource, positions_source: ReadOnlySource, - delete_bitset: DeleteBitSet, + delete_bitset_opt: Option, record_option: IndexRecordOption, } @@ -34,14 +35,14 @@ impl InvertedIndexReader { termdict_source: ReadOnlySource, postings_source: ReadOnlySource, positions_source: ReadOnlySource, - delete_bitset: DeleteBitSet, + delete_bitset_opt: Option, record_option: IndexRecordOption, ) -> InvertedIndexReader { InvertedIndexReader { termdict: TermDictionaryImpl::from_source(termdict_source), postings_source, positions_source, - delete_bitset, + delete_bitset_opt, record_option, } } @@ -105,13 +106,15 @@ impl InvertedIndexReader { /// This method is for an advanced usage only. /// /// Most user should prefer using `read_postings` instead. - pub fn read_postings_from_terminfo( + pub fn read_postings_from_terminfo( &self, term_info: &TermInfo, option: IndexRecordOption, - ) -> SegmentPostings { + ) -> SegmentPostings { let block_postings = self.read_block_postings_from_terminfo(term_info, option); - let delete_bitset = self.delete_bitset.clone(); + let delete_set = TDeleteSet::from(self.delete_bitset_opt.iter() + .cloned() + .next()); let position_stream = { if option.has_positions() { let position_offset = term_info.positions_offset; @@ -123,7 +126,7 @@ impl InvertedIndexReader { None } }; - SegmentPostings::from_block_postings(block_postings, delete_bitset, position_stream) + SegmentPostings::from_block_postings(block_postings, delete_set, position_stream) } /// Returns the segment postings associated with the term, and with the given option, @@ -136,11 +139,17 @@ impl InvertedIndexReader { /// For instance, requesting `IndexRecordOption::Freq` for a /// `TextIndexingOptions` that does not index position will return a `SegmentPostings` /// with `DocId`s and frequencies. - pub fn read_postings(&self, term: &Term, option: IndexRecordOption) -> Option { + pub fn read_postings(&self, term: &Term, option: IndexRecordOption) -> Option> { let term_info = get!(self.get_term_info(term)); Some(self.read_postings_from_terminfo(&term_info, option)) } + pub(crate) fn read_postings_no_deletes(&self, term: &Term, option: IndexRecordOption) -> Option> { + let term_info = get!(self.get_term_info(term)); + Some(self.read_postings_from_terminfo(&term_info, option)) + } + + /// Returns the number of documents containing the term. pub fn doc_freq(&self, term: &Term) -> u32 { self.get_term_info(term) diff --git a/src/core/segment_reader.rs b/src/core/segment_reader.rs index 85ddbfea0..670223812 100644 --- a/src/core/segment_reader.rs +++ b/src/core/segment_reader.rs @@ -26,6 +26,7 @@ use schema::Schema; use termdict::TermDictionary; use fastfield::{FastValue, MultiValueIntFastFieldReader}; use schema::Cardinality; +use postings::DeleteSet; /// Entry point to access all of the datastructures of the `Segment` /// @@ -54,7 +55,7 @@ pub struct SegmentReader { fieldnorms_composite: CompositeFile, store_reader: StoreReader, - delete_bitset: DeleteBitSet, + delete_bitset_opt: Option, schema: Schema, } @@ -79,7 +80,13 @@ impl SegmentReader { /// Return the number of documents that have been /// deleted in the segment. pub fn num_deleted_docs(&self) -> DocId { - self.delete_bitset.len() as DocId + self.delete_bitset() + .map(|delete_set| delete_set.len() as DocId) + .unwrap_or(0u32) + } + + pub fn has_deletes(&self) -> bool { + self.num_deleted_docs() > 0 } /// Accessor to a segment's fast field reader given a field. @@ -194,12 +201,13 @@ impl SegmentReader { let fieldnorms_data = segment.open_read(SegmentComponent::FIELDNORMS)?; let fieldnorms_composite = CompositeFile::open(&fieldnorms_data)?; - let delete_bitset = if segment.meta().has_deletes() { - let delete_data = segment.open_read(SegmentComponent::DELETE)?; - DeleteBitSet::open(delete_data) - } else { - DeleteBitSet::empty() - }; + let delete_bitset_opt = + if segment.meta().has_deletes() { + let delete_data = segment.open_read(SegmentComponent::DELETE)?; + Some(DeleteBitSet::open(delete_data)) + } else { + None + }; let schema = segment.schema(); Ok(SegmentReader { @@ -211,7 +219,7 @@ impl SegmentReader { fieldnorms_composite, segment_id: segment.id(), store_reader, - delete_bitset, + delete_bitset_opt, positions_composite, schema, }) @@ -253,7 +261,7 @@ impl SegmentReader { termdict_source, postings_source, positions_source, - self.delete_bitset.clone(), + self.delete_bitset_opt.clone(), record_option, )); @@ -282,14 +290,16 @@ impl SegmentReader { /// Returns the bitset representing /// the documents that have been deleted. - pub fn delete_bitset(&self) -> &DeleteBitSet { - &self.delete_bitset + pub fn delete_bitset(&self) -> Option<&DeleteBitSet> { + self.delete_bitset_opt.as_ref() } /// Returns true iff the `doc` is marked /// as deleted. pub fn is_deleted(&self, doc: DocId) -> bool { - self.delete_bitset.is_deleted(doc) + self.delete_bitset() + .map(|delete_set| delete_set.is_deleted(doc)) + .unwrap_or(false) } } diff --git a/src/fastfield/delete.rs b/src/fastfield/delete.rs index 02634ed0f..3134c3fc1 100644 --- a/src/fastfield/delete.rs +++ b/src/fastfield/delete.rs @@ -5,6 +5,7 @@ use std::io; use directory::ReadOnlySource; use DocId; use common::HasLen; +use postings::DeleteSet; /// Write a delete `BitSet` /// @@ -51,22 +52,19 @@ impl DeleteBitSet { } } - /// Returns an empty delete bit set. - pub fn empty() -> DeleteBitSet { +} + + +impl DeleteSet for DeleteBitSet { + + fn empty() -> DeleteBitSet { DeleteBitSet { data: ReadOnlySource::empty(), len: 0, } } - /// Returns true iff the segment has some deleted documents. - pub fn has_deletes(&self) -> bool { - self.len() > 0 - } - - /// Returns true iff the document is deleted. - #[inline] - pub fn is_deleted(&self, doc: DocId) -> bool { + fn is_deleted(&self, doc: DocId) -> bool { if self.len == 0 { false } else { @@ -76,6 +74,18 @@ impl DeleteBitSet { b & (1u8 << shift) != 0 } } + +} + + +impl From> for DeleteBitSet { + fn from(delete_bitset_opt: Option) -> Self { + if let Some(delete_bitset) = delete_bitset_opt { + delete_bitset + } else { + DeleteBitSet::empty() + } + } } impl HasLen for DeleteBitSet { diff --git a/src/indexer/merger.rs b/src/indexer/merger.rs index 7e1a0580b..57f03a80a 100644 --- a/src/indexer/merger.rs +++ b/src/indexer/merger.rs @@ -17,6 +17,7 @@ use store::StoreWriter; use std::cmp::{max, min}; use termdict::TermDictionary; use termdict::TermStreamer; +use postings::DeleteSet; pub struct IndexMerger { schema: Schema, @@ -27,22 +28,28 @@ pub struct IndexMerger { fn compute_min_max_val( u64_reader: &FastFieldReader, max_doc: DocId, - delete_bitset: &DeleteBitSet, + delete_bitset_opt: Option<&DeleteBitSet>, ) -> Option<(u64, u64)> { if max_doc == 0 { None - } else if !delete_bitset.has_deletes() { - // no deleted documents, - // we can use the previous min_val, max_val. - Some((u64_reader.min_value(), u64_reader.max_value())) } else { - // some deleted documents, - // we need to recompute the max / min - (0..max_doc) - .filter(|doc_id| !delete_bitset.is_deleted(*doc_id)) - .map(|doc_id| u64_reader.get(doc_id)) - .minmax() - .into_option() + match delete_bitset_opt { + Some(delete_bitset) => { + // some deleted documents, + // we need to recompute the max / min + (0..max_doc) + .filter(|doc_id| !delete_bitset.is_deleted(*doc_id)) + .map(|doc_id| u64_reader.get(doc_id)) + .minmax() + .into_option() + + } + None => { + // no deleted documents, + // we can use the previous min_val, max_val. + Some((u64_reader.min_value(), u64_reader.max_value())) + } + } } } @@ -150,7 +157,7 @@ impl IndexMerger { if let Some((seg_min_val, seg_max_val)) = compute_min_max_val( &u64_reader, reader.max_doc(), - reader.delete_bitset(), + reader.delete_bitset() ) { // the segment has some non-deleted documents min_val = min(min_val, seg_min_val); @@ -181,12 +188,15 @@ impl IndexMerger { let mut fast_single_field_serializer = fast_field_serializer.new_u64_fast_field(field, min_val, max_val)?; - for (max_doc, u64_reader, delete_bitset) in u64_readers { + for (max_doc, u64_reader, delete_bitset_opt) in u64_readers { for doc_id in 0..max_doc { - if !delete_bitset.is_deleted(doc_id) { - let val = u64_reader.get(doc_id); - fast_single_field_serializer.add_val(val)?; + if let Some(ref delete_bitset) = delete_bitset_opt { + if delete_bitset.is_deleted(doc_id) { + continue; + } } + let val = u64_reader.get(doc_id); + fast_single_field_serializer.add_val(val)?; } } @@ -271,7 +281,7 @@ impl IndexMerger { let segment_reader = &self.readers[heap_item.segment_ord]; let inverted_index = segment_reader.inverted_index(indexed_field); let mut segment_postings = inverted_index - .read_postings_from_terminfo(term_info, segment_postings_option); + .read_postings_from_terminfo::(term_info, segment_postings_option); if segment_postings.advance() { Some((segment_ord, segment_postings)) } else { diff --git a/src/postings/delete_set.rs b/src/postings/delete_set.rs new file mode 100644 index 000000000..569611473 --- /dev/null +++ b/src/postings/delete_set.rs @@ -0,0 +1,27 @@ +use fastfield::DeleteBitSet; +use DocId; + +pub trait DeleteSet: 'static + From> { + fn is_deleted(&self, doc: DocId) -> bool; + fn empty() -> Self; +} + + +#[derive(Default)] +pub struct NoDelete; +impl DeleteSet for NoDelete { + #[inline(always)] + fn is_deleted(&self, doc: DocId) -> bool { + false + } + fn empty() -> Self { + NoDelete + } +} + +impl From> for NoDelete { + fn from(delete_bitset_opt: Option) -> Self { + assert!(delete_bitset_opt.is_none(), "NoDelete should not be used if there are some deleted documents."); + NoDelete + } +} \ No newline at end of file diff --git a/src/postings/mod.rs b/src/postings/mod.rs index c67a9f855..31e7cc6cc 100644 --- a/src/postings/mod.rs +++ b/src/postings/mod.rs @@ -13,11 +13,13 @@ mod serializer; mod postings_writer; mod term_info; mod segment_postings; +mod delete_set; use self::recorder::{NothingRecorder, Recorder, TFAndPositionRecorder, TermFrequencyRecorder}; pub use self::serializer::{FieldSerializer, InvertedIndexSerializer}; pub(crate) use self::postings_writer::MultiFieldPostingsWriter; +pub use self::delete_set::{DeleteSet, NoDelete}; pub use self::term_info::TermInfo; pub use self::postings::Postings; @@ -42,7 +44,7 @@ pub mod tests { use DocId; use Score; use query::Intersection; - use query::Scorer; + use query::{Weight, Scorer}; use schema::{Document, SchemaBuilder, Term, INT_INDEXED, STRING, TEXT}; use core::SegmentComponent; use indexer::SegmentWriter; @@ -97,53 +99,54 @@ pub mod tests { index_writer.add_document(doc!(title => r#"abc be be be be abc"#)); index_writer.commit().unwrap(); index.load_searchers().unwrap(); + let searcher = index.searcher(); - let query = TermQuery::new( - Term::from_field_text(title, "abc"), - IndexRecordOption::WithFreqsAndPositions, - ); - let weight = query.specialized_weight(&*searcher, true); + let inverted_index = searcher.segment_reader(0u32).inverted_index(title); + let term = Term::from_field_text(title, "abc"); + + { - let mut scorer = weight - .specialized_scorer(searcher.segment_reader(0u32)) + let mut postings = inverted_index + .read_postings(&term, IndexRecordOption::WithFreqsAndPositions) .unwrap(); - scorer.advance(); - assert_eq!(&[0, 1, 2], scorer.postings().positions()); - scorer.advance(); - assert_eq!(&[0, 5], scorer.postings().positions()); + postings.advance(); + assert_eq!(&[0, 1, 2], postings.positions()); + postings.advance(); + assert_eq!(&[0, 5], postings.positions()); } { - let mut scorer = weight - .specialized_scorer(searcher.segment_reader(0u32)) + let mut postings = inverted_index + .read_postings(&term, IndexRecordOption::WithFreqsAndPositions) .unwrap(); - scorer.advance(); - scorer.advance(); - assert_eq!(&[0, 5], scorer.postings().positions()); + postings.advance(); + postings.advance(); + assert_eq!(&[0, 5], postings.positions()); } { - let mut scorer = weight - .specialized_scorer(searcher.segment_reader(0u32)) + + let mut postings = inverted_index + .read_postings(&term, IndexRecordOption::WithFreqsAndPositions) .unwrap(); - assert_eq!(scorer.skip_next(1), SkipResult::Reached); - assert_eq!(scorer.doc(), 1); - assert_eq!(&[0, 5], scorer.postings().positions()); + assert_eq!(postings.skip_next(1), SkipResult::Reached); + assert_eq!(postings.doc(), 1); + assert_eq!(&[0, 5], postings.positions()); } { - let mut scorer = weight - .specialized_scorer(searcher.segment_reader(0u32)) + let mut postings = inverted_index + .read_postings(&term, IndexRecordOption::WithFreqsAndPositions) .unwrap(); - assert_eq!(scorer.skip_next(1002), SkipResult::Reached); - assert_eq!(scorer.doc(), 1002); - assert_eq!(&[0, 5], scorer.postings().positions()); + assert_eq!(postings.skip_next(1002), SkipResult::Reached); + assert_eq!(postings.doc(), 1002); + assert_eq!(&[0, 5], postings.positions()); } { - let mut scorer = weight - .specialized_scorer(searcher.segment_reader(0u32)) + let mut postings = inverted_index + .read_postings(&term, IndexRecordOption::WithFreqsAndPositions) .unwrap(); - assert_eq!(scorer.skip_next(100), SkipResult::Reached); - assert_eq!(scorer.skip_next(1002), SkipResult::Reached); - assert_eq!(scorer.doc(), 1002); - assert_eq!(&[0, 5], scorer.postings().positions()); + assert_eq!(postings.skip_next(100), SkipResult::Reached); + assert_eq!(postings.skip_next(1002), SkipResult::Reached); + assert_eq!(postings.doc(), 1002); + assert_eq!(&[0, 5], postings.positions()); } } @@ -277,18 +280,16 @@ pub mod tests { assert!(index_writer.commit().is_ok()); } index.load_searchers().unwrap(); - let term_query = TermQuery::new( - Term::from_field_text(text_field, "a"), - IndexRecordOption::Basic, - ); + let term_a = Term::from_field_text(text_field, "a"); let searcher = index.searcher(); - let mut term_weight = term_query.specialized_weight(&*searcher, true); - term_weight.index_record_option = IndexRecordOption::WithFreqsAndPositions; - let segment_reader = &searcher.segment_readers()[0]; - let mut term_scorer = term_weight.specialized_scorer(segment_reader).unwrap(); - assert!(term_scorer.advance()); - assert_eq!(term_scorer.doc(), 1u32); - assert_eq!(term_scorer.postings().positions(), &[1u32, 4]); + let segment_reader = searcher.segment_reader(0); + let mut postings = segment_reader + .inverted_index(text_field) + .read_postings(&term_a, IndexRecordOption::WithFreqsAndPositions) + .unwrap(); + assert!(postings.advance()); + assert_eq!(postings.doc(), 1u32); + assert_eq!(postings.positions(), &[1u32, 4]); } #[test] diff --git a/src/postings/segment_postings.rs b/src/postings/segment_postings.rs index 239e8c306..bdeddbecf 100644 --- a/src/postings/segment_postings.rs +++ b/src/postings/segment_postings.rs @@ -8,6 +8,7 @@ use docset::{DocSet, SkipResult}; use std::cmp; use fst::Streamer; use compression::compressed_block_size; +use postings::{NoDelete, DeleteSet}; use fastfield::DeleteBitSet; use std::cell::UnsafeCell; use directory::{ReadOnlySource, SourceRead}; @@ -66,14 +67,25 @@ impl PositionComputer { /// /// As we iterate through the `SegmentPostings`, the frequencies are optionally decoded. /// Positions on the other hand, are optionally entirely decoded upfront. -pub struct SegmentPostings { +pub struct SegmentPostings { block_cursor: BlockSegmentPostings, cur: usize, - delete_bitset: DeleteBitSet, + delete_bitset: TDeleteSet, position_computer: Option>, } -impl SegmentPostings { +impl SegmentPostings { + /// Returns an empty segment postings object + pub fn empty() -> Self { + let empty_block_cursor = BlockSegmentPostings::empty(); + SegmentPostings { + block_cursor: empty_block_cursor, + delete_bitset: NoDelete, + cur: COMPRESSION_BLOCK_SIZE, + position_computer: None, + } + } + /// Creates a segment postings object with the given documents /// and no frequency encoded. /// @@ -82,14 +94,14 @@ impl SegmentPostings { /// It serializes the doc ids using tantivy's codec /// and returns a `SegmentPostings` object that embeds a /// buffer with the serialized data. - pub fn create_from_docs(docs: &[u32]) -> SegmentPostings { + pub fn create_from_docs(docs: &[u32]) -> SegmentPostings { let mut buffer = Vec::new(); { let mut postings_serializer = PostingsSerializer::new(&mut buffer, false); for &doc in docs { postings_serializer.write_doc(doc, 1u32).unwrap(); } - postings_serializer.close_term().unwrap(); + postings_serializer.close_term().expect("In memory Serialization should never fail."); } let data = ReadOnlySource::from(buffer); let block_segment_postings = BlockSegmentPostings::from_data( @@ -97,8 +109,20 @@ impl SegmentPostings { SourceRead::from(data), FreqReadingOption::NoFreq, ); - SegmentPostings::from_block_postings(block_segment_postings, DeleteBitSet::empty(), None) + SegmentPostings::from_block_postings(block_segment_postings, NoDelete, None) } +} + +impl SegmentPostings { + fn position_add_skip usize>(&self, num_skips_fn: F) { + if let Some(position_computer) = self.position_computer.as_ref() { + let num_skips = num_skips_fn(); + unsafe { + (*position_computer.get()).add_skip(num_skips); + } + } + } + /// Reads a Segment postings from an &[u8] /// @@ -108,9 +132,9 @@ impl SegmentPostings { /// frequencies and/or positions pub fn from_block_postings( segment_block_postings: BlockSegmentPostings, - delete_bitset: DeleteBitSet, + delete_bitset: TDeleteSet, positions_stream_opt: Option, - ) -> SegmentPostings { + ) -> SegmentPostings { let position_computer = positions_stream_opt.map(|stream| UnsafeCell::new(PositionComputer::new(stream))); SegmentPostings { @@ -120,29 +144,9 @@ impl SegmentPostings { position_computer, } } - - /// Returns an empty segment postings object - pub fn empty() -> SegmentPostings { - let empty_block_cursor = BlockSegmentPostings::empty(); - SegmentPostings { - block_cursor: empty_block_cursor, - delete_bitset: DeleteBitSet::empty(), - cur: COMPRESSION_BLOCK_SIZE, - position_computer: None, - } - } - - fn position_add_skip usize>(&self, num_skips_fn: F) { - if let Some(position_computer) = self.position_computer.as_ref() { - let num_skips = num_skips_fn(); - unsafe { - (*position_computer.get()).add_skip(num_skips); - } - } - } } -impl DocSet for SegmentPostings { +impl DocSet for SegmentPostings { // goes to the next element. // next needs to be called a first time to point to the correct element. #[inline] @@ -299,13 +303,14 @@ impl DocSet for SegmentPostings { } } -impl HasLen for SegmentPostings { + +impl HasLen for SegmentPostings { fn len(&self) -> usize { self.block_cursor.doc_freq() } } -impl Postings for SegmentPostings { +impl Postings for SegmentPostings { fn term_freq(&self) -> u32 { self.block_cursor.freq(self.cur) } diff --git a/src/query/boolean_query/boolean_weight.rs b/src/query/boolean_query/boolean_weight.rs index 25f56693b..f2fa7c18f 100644 --- a/src/query/boolean_query/boolean_weight.rs +++ b/src/query/boolean_query/boolean_weight.rs @@ -10,6 +10,9 @@ use std::borrow::Borrow; use query::Exclude; use query::Occur; use query::RequiredOptionalScorer; +use query::IntersectionTwoTerms; +use fastfield::DeleteBitSet; +use postings::NoDelete; use query::score_combiner::{DoNothingCombiner, ScoreCombiner, SumWithCoordsCombiner}; use Result; @@ -19,24 +22,41 @@ where { assert!(!scorers.is_empty()); if scorers.len() == 1 { - scorers.into_iter().next().unwrap() //< we checked the size beforehands - } else { + return scorers.into_iter().next().unwrap(); //< we checked the size beforehands + } + + { let is_all_term_queries = scorers.iter().all(|scorer| { let scorer_ref: &Scorer = scorer.borrow(); - Downcast::::is_type(scorer_ref) + Downcast::>::is_type(scorer_ref) }); if is_all_term_queries { - let scorers: Vec = scorers + let scorers: Vec> = scorers .into_iter() - .map(|scorer| *Downcast::::downcast(scorer).unwrap()) + .map(|scorer| *Downcast::>::downcast(scorer).unwrap()) .collect(); - let scorer: Box = box Union::::from(scorers); - scorer - } else { - let scorer: Box = box Union::<_, TScoreCombiner>::from(scorers); - scorer + let scorer: Box = box Union::, TScoreCombiner>::from(scorers); + return scorer; } } + + { + let is_all_term_queries = scorers.iter().all(|scorer| { + let scorer_ref: &Scorer = scorer.borrow(); + Downcast::>::is_type(scorer_ref) + }); + if is_all_term_queries { + let scorers: Vec> = scorers + .into_iter() + .map(|scorer| *Downcast::>::downcast(scorer).unwrap()) + .collect(); + let scorer: Box = box Union::, TScoreCombiner>::from(scorers); + return scorer; + } + } + let scorer: Box = box Union::<_, TScoreCombiner>::from(scorers); + return scorer; + } pub struct BooleanWeight { @@ -74,26 +94,55 @@ impl BooleanWeight { .map(scorer_union::); let must_scorer_opt: Option> = - per_occur_scorers.remove(&Occur::Must).map(|scorers| { + per_occur_scorers.remove(&Occur::Must).map(|mut scorers| { if scorers.len() == 1 { - scorers.into_iter().next().unwrap() - } else { + return scorers.into_iter().next().unwrap(); + } + scorers.sort_by_key(|scorer| scorer.size_hint()); + { let is_all_term_queries = scorers.iter().all(|scorer| { let scorer_ref: &Scorer = scorer.borrow(); - Downcast::::is_type(scorer_ref) + Downcast::>::is_type(scorer_ref) }); if is_all_term_queries { - let scorers: Vec = scorers - .into_iter() - .map(|scorer| *Downcast::::downcast(scorer).unwrap()) - .collect(); - let scorer: Box = box Intersection::from(scorers); - scorer - } else { - let scorer: Box = box Intersection::from(scorers); - scorer + if scorers.len() == 2 { + let right = scorers.pop().unwrap(); + let left = scorers.pop().unwrap(); + return box IntersectionTwoTerms::new(left, right); + } else { + let mut scorers: Vec> = scorers + .into_iter() + .map(|scorer| *Downcast::>::downcast(scorer).unwrap()) + .collect(); + let scorer: Box = box Intersection::from(scorers); + return scorer; + } } } + { + let is_all_term_queries = scorers.iter().all(|scorer| { + let scorer_ref: &Scorer = scorer.borrow(); + Downcast::>::is_type(scorer_ref) + }); + if is_all_term_queries { + if scorers.len() == 2 { + let right = scorers.pop().unwrap(); + let left = scorers.pop().unwrap(); + return box IntersectionTwoTerms::new(left, right); + } else { + let mut scorers: Vec> = scorers + .into_iter() + .map(|scorer| *Downcast::>::downcast(scorer).unwrap()) + .collect(); + let scorer: Box = box Intersection::from(scorers); + return scorer; + } + } + } + { + let scorer: Box = box Intersection::from(scorers); + scorer + } }); let positive_scorer: Box = match (should_scorer_opt, must_scorer_opt) { diff --git a/src/query/boolean_query/mod.rs b/src/query/boolean_query/mod.rs index fd7393050..ca78939ad 100644 --- a/src/query/boolean_query/mod.rs +++ b/src/query/boolean_query/mod.rs @@ -85,7 +85,7 @@ mod tests { let query = query_parser.parse_query("+a +b +c").unwrap(); let weight = query.weight(&*searcher, true).unwrap(); let scorer = weight.scorer(searcher.segment_reader(0u32)).unwrap(); - assert!(Downcast::>::is_type(&*scorer)); + assert!(Downcast::>>::is_type(&*scorer)); } { let query = query_parser.parse_query("+a +(b c)").unwrap(); @@ -95,6 +95,8 @@ mod tests { } } + use postings::NoDelete; + #[test] pub fn test_boolean_reqopt() { let (index, text_field) = aux_test_helper(); @@ -110,7 +112,7 @@ mod tests { let query = query_parser.parse_query("+a b").unwrap(); let weight = query.weight(&*searcher, false).unwrap(); let scorer = weight.scorer(searcher.segment_reader(0u32)).unwrap(); - assert!(Downcast::::is_type(&*scorer)); + assert!(Downcast::>::is_type(&*scorer)); } } diff --git a/src/query/intersection_two.rs b/src/query/intersection_two.rs new file mode 100644 index 000000000..59a7a2366 --- /dev/null +++ b/src/query/intersection_two.rs @@ -0,0 +1,73 @@ +use docset::DocSet; +use query::Scorer; +use DocId; +use Score; +use std::mem; +use query::term_query::TermScorer; +use postings::DeleteSet; +use SkipResult; + + +/// Creates a `DocSet` that iterator through the intersection of two `DocSet`s. +pub struct IntersectionTwoTerms { + left: TDocSet, + right: TDocSet +} + +impl IntersectionTwoTerms { + pub fn new(left: TDocSet, mut right: TDocSet) -> IntersectionTwoTerms { + IntersectionTwoTerms { + left, + right + } + } +} + +impl DocSet for IntersectionTwoTerms { + + fn advance(&mut self) -> bool { + let (left, right) = (&mut self.left, &mut self.right); + if !left.advance() { + return false; + } + let mut candidate = left.doc(); + loop { + match right.skip_next(candidate) { + SkipResult::Reached => { + return true; + } + SkipResult::End => { + return false; + } + SkipResult::OverStep => { + candidate = right.doc(); + } + } + match left.skip_next(candidate) { + SkipResult::Reached => { + return true; + } + SkipResult::End => { + return false; + } + SkipResult::OverStep => { + candidate = left.doc(); + } + } + } + } + + fn doc(&self) -> DocId { + self.left.doc() + } + + fn size_hint(&self) -> u32 { + self.left.size_hint().min(self.right.size_hint()) + } +} + +impl Scorer for IntersectionTwoTerms { + fn score(&mut self) -> Score { + self.left.score() + self.right.score() + } +} diff --git a/src/query/mod.rs b/src/query/mod.rs index 1872372a9..df68004c2 100644 --- a/src/query/mod.rs +++ b/src/query/mod.rs @@ -16,6 +16,7 @@ mod range_query; mod exclude; mod union; mod intersection; +mod intersection_two; mod reqopt_scorer; #[cfg(test)] @@ -45,3 +46,4 @@ pub use self::weight::Weight; pub use self::all_query::{AllQuery, AllScorer, AllWeight}; pub use self::range_query::RangeQuery; pub use self::scorer::ConstScorer; +pub use self::intersection_two::IntersectionTwoTerms; diff --git a/src/query/phrase_query/phrase_scorer.rs b/src/query/phrase_query/phrase_scorer.rs index 47ae7d64e..a58dda6a5 100644 --- a/src/query/phrase_query/phrase_scorer.rs +++ b/src/query/phrase_query/phrase_scorer.rs @@ -2,14 +2,15 @@ use DocId; use docset::{DocSet, SkipResult}; use postings::{Postings, SegmentPostings}; use query::{Intersection, Scorer}; +use fastfield::DeleteBitSet; struct PostingsWithOffset { offset: u32, - segment_postings: SegmentPostings, + segment_postings: SegmentPostings, } impl PostingsWithOffset { - pub fn new(segment_postings: SegmentPostings, offset: u32) -> PostingsWithOffset { + pub fn new(segment_postings: SegmentPostings, offset: u32) -> PostingsWithOffset { PostingsWithOffset { offset, segment_postings, @@ -50,7 +51,7 @@ pub struct PhraseScorer { } impl PhraseScorer { - pub fn new(term_postings: Vec) -> PhraseScorer { + pub fn new(term_postings: Vec>) -> PhraseScorer { let postings_with_offsets: Vec<_> = term_postings .into_iter() .enumerate() diff --git a/src/query/term_query/term_scorer.rs b/src/query/term_query/term_scorer.rs index d8352780c..9bb0763e3 100644 --- a/src/query/term_query/term_scorer.rs +++ b/src/query/term_query/term_scorer.rs @@ -5,20 +5,21 @@ use postings::SegmentPostings; use query::Scorer; use postings::Postings; use fastfield::FastFieldReader; +use postings::{NoDelete, DeleteSet}; -pub struct TermScorer { +pub struct TermScorer { pub idf: Score, pub fieldnorm_reader_opt: Option>, - pub postings: SegmentPostings, + pub postings: SegmentPostings, } -impl TermScorer { - pub fn postings(&self) -> &SegmentPostings { +impl TermScorer { + pub fn postings(&self) -> &SegmentPostings { &self.postings } } -impl DocSet for TermScorer { +impl DocSet for TermScorer { fn advance(&mut self) -> bool { self.postings.advance() } @@ -36,7 +37,7 @@ impl DocSet for TermScorer { } } -impl Scorer for TermScorer { +impl Scorer for TermScorer { fn score(&mut self) -> Score { let doc = self.postings.doc(); let tf = match self.fieldnorm_reader_opt { diff --git a/src/query/term_query/term_weight.rs b/src/query/term_query/term_weight.rs index 224718ab1..2b2da02df 100644 --- a/src/query/term_query/term_weight.rs +++ b/src/query/term_query/term_weight.rs @@ -6,6 +6,8 @@ use docset::DocSet; use postings::SegmentPostings; use schema::IndexRecordOption; use super::term_scorer::TermScorer; +use fastfield::DeleteBitSet; +use postings::{DeleteSet, NoDelete}; use Result; pub struct TermWeight { @@ -17,8 +19,46 @@ pub struct TermWeight { impl Weight for TermWeight { fn scorer(&self, reader: &SegmentReader) -> Result> { - let specialized_scorer = self.specialized_scorer(reader)?; - Ok(box specialized_scorer) + let field = self.term.field(); + let inverted_index = reader.inverted_index(field); + let fieldnorm_reader_opt = reader.get_fieldnorms_reader(field); + let scorer: Box; + if reader.has_deletes() { + let postings_opt: Option> = + inverted_index.read_postings(&self.term, self.index_record_option); + scorer = + if let Some(segment_postings) = postings_opt { + box TermScorer { + idf: self.idf(), + fieldnorm_reader_opt, + postings: segment_postings, + } + } else { + box TermScorer { + idf: 1f32, + fieldnorm_reader_opt: None, + postings: SegmentPostings::::empty(), + } + }; + } else { + let postings_opt: Option> = + inverted_index.read_postings_no_deletes(&self.term, self.index_record_option); + scorer = + if let Some(segment_postings) = postings_opt { + box TermScorer { + idf: self.idf(), + fieldnorm_reader_opt, + postings: segment_postings, + } + } else { + box TermScorer { + idf: 1f32, + fieldnorm_reader_opt: None, + postings: SegmentPostings::::empty(), + } + }; + } + Ok(scorer) } fn count(&self, reader: &SegmentReader) -> Result { @@ -30,7 +70,7 @@ impl Weight for TermWeight { .map(|term_info| term_info.doc_freq) .unwrap_or(0)) } else { - Ok(self.specialized_scorer(reader)?.count()) + Ok(self.scorer(reader)?.count()) } } } @@ -39,26 +79,4 @@ impl TermWeight { fn idf(&self) -> f32 { 1.0 + (self.num_docs as f32 / (self.doc_freq as f32 + 1.0)).ln() } - - /// If the field is not found, returns an empty `DocSet`. - pub fn specialized_scorer(&self, reader: &SegmentReader) -> Result { - let field = self.term.field(); - let inverted_index = reader.inverted_index(field); - let fieldnorm_reader_opt = reader.get_fieldnorms_reader(field); - let postings_opt: Option = - inverted_index.read_postings(&self.term, self.index_record_option); - if let Some(segment_postings) = postings_opt { - Ok(TermScorer { - idf: self.idf(), - fieldnorm_reader_opt, - postings: segment_postings, - }) - } else { - Ok(TermScorer { - idf: 1f32, - fieldnorm_reader_opt: None, - postings: SegmentPostings::empty(), - }) - } - } } From 4ee2db25a0b640be638da642a359126b3a5f1780 Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Thu, 22 Feb 2018 08:26:45 +0900 Subject: [PATCH 02/24] Generic on Postings rather than deletes in TermScorer --- src/postings/delete_set.rs | 2 +- src/postings/mod.rs | 3 +-- src/postings/postings.rs | 2 +- src/postings/segment_postings.rs | 1 - src/query/boolean_query/boolean_weight.rs | 33 +++++++++++------------ src/query/boolean_query/mod.rs | 13 ++++----- src/query/intersection_two.rs | 5 +--- src/query/term_query/mod.rs | 7 +++++ src/query/term_query/term_scorer.rs | 18 ++++--------- src/query/term_query/term_weight.rs | 2 +- 10 files changed, 38 insertions(+), 48 deletions(-) diff --git a/src/postings/delete_set.rs b/src/postings/delete_set.rs index 569611473..068863e68 100644 --- a/src/postings/delete_set.rs +++ b/src/postings/delete_set.rs @@ -11,7 +11,7 @@ pub trait DeleteSet: 'static + From> { pub struct NoDelete; impl DeleteSet for NoDelete { #[inline(always)] - fn is_deleted(&self, doc: DocId) -> bool { + fn is_deleted(&self, _doc: DocId) -> bool { false } fn empty() -> Self { diff --git a/src/postings/mod.rs b/src/postings/mod.rs index 31e7cc6cc..2296a4e3e 100644 --- a/src/postings/mod.rs +++ b/src/postings/mod.rs @@ -44,7 +44,7 @@ pub mod tests { use DocId; use Score; use query::Intersection; - use query::{Weight, Scorer}; + use query::Scorer; use schema::{Document, SchemaBuilder, Term, INT_INDEXED, STRING, TEXT}; use core::SegmentComponent; use indexer::SegmentWriter; @@ -53,7 +53,6 @@ pub mod tests { use schema::IndexRecordOption; use std::iter; use datastruct::stacker::Heap; - use query::TermQuery; use schema::Field; use test::{self, Bencher}; use indexer::operation::AddOperation; diff --git a/src/postings/postings.rs b/src/postings/postings.rs index aceb600be..92056b958 100644 --- a/src/postings/postings.rs +++ b/src/postings/postings.rs @@ -10,7 +10,7 @@ use docset::DocSet; /// Its main implementation is `SegmentPostings`, /// but other implementations mocking `SegmentPostings` exist, /// for merging segments or for testing. -pub trait Postings: DocSet { +pub trait Postings: DocSet + 'static { /// Returns the term frequency fn term_freq(&self) -> u32; /// Returns the list of positions of the term, expressed as a list of diff --git a/src/postings/segment_postings.rs b/src/postings/segment_postings.rs index bdeddbecf..5531c944e 100644 --- a/src/postings/segment_postings.rs +++ b/src/postings/segment_postings.rs @@ -9,7 +9,6 @@ use std::cmp; use fst::Streamer; use compression::compressed_block_size; use postings::{NoDelete, DeleteSet}; -use fastfield::DeleteBitSet; use std::cell::UnsafeCell; use directory::{ReadOnlySource, SourceRead}; use postings::FreqReadingOption; diff --git a/src/query/boolean_query/boolean_weight.rs b/src/query/boolean_query/boolean_weight.rs index f2fa7c18f..51a3412f9 100644 --- a/src/query/boolean_query/boolean_weight.rs +++ b/src/query/boolean_query/boolean_weight.rs @@ -5,16 +5,15 @@ use std::collections::HashMap; use query::EmptyScorer; use query::Scorer; use downcast::Downcast; -use query::term_query::TermScorer; use std::borrow::Borrow; use query::Exclude; use query::Occur; use query::RequiredOptionalScorer; use query::IntersectionTwoTerms; -use fastfield::DeleteBitSet; -use postings::NoDelete; use query::score_combiner::{DoNothingCombiner, ScoreCombiner, SumWithCoordsCombiner}; use Result; +use query::term_query::{TermScorerWithDeletes, TermScorerNoDeletes}; + fn scorer_union(scorers: Vec>) -> Box where @@ -28,14 +27,14 @@ where { let is_all_term_queries = scorers.iter().all(|scorer| { let scorer_ref: &Scorer = scorer.borrow(); - Downcast::>::is_type(scorer_ref) + Downcast::::is_type(scorer_ref) }); if is_all_term_queries { - let scorers: Vec> = scorers + let scorers: Vec = scorers .into_iter() - .map(|scorer| *Downcast::>::downcast(scorer).unwrap()) + .map(|scorer| *Downcast::::downcast(scorer).unwrap()) .collect(); - let scorer: Box = box Union::, TScoreCombiner>::from(scorers); + let scorer: Box = box Union::::from(scorers); return scorer; } } @@ -43,14 +42,14 @@ where { let is_all_term_queries = scorers.iter().all(|scorer| { let scorer_ref: &Scorer = scorer.borrow(); - Downcast::>::is_type(scorer_ref) + Downcast::::is_type(scorer_ref) }); if is_all_term_queries { - let scorers: Vec> = scorers + let scorers: Vec = scorers .into_iter() - .map(|scorer| *Downcast::>::downcast(scorer).unwrap()) + .map(|scorer| *Downcast::::downcast(scorer).unwrap()) .collect(); - let scorer: Box = box Union::, TScoreCombiner>::from(scorers); + let scorer: Box = box Union::::from(scorers); return scorer; } } @@ -102,7 +101,7 @@ impl BooleanWeight { { let is_all_term_queries = scorers.iter().all(|scorer| { let scorer_ref: &Scorer = scorer.borrow(); - Downcast::>::is_type(scorer_ref) + Downcast::::is_type(scorer_ref) }); if is_all_term_queries { if scorers.len() == 2 { @@ -110,9 +109,9 @@ impl BooleanWeight { let left = scorers.pop().unwrap(); return box IntersectionTwoTerms::new(left, right); } else { - let mut scorers: Vec> = scorers + let scorers: Vec = scorers .into_iter() - .map(|scorer| *Downcast::>::downcast(scorer).unwrap()) + .map(|scorer| *Downcast::::downcast(scorer).unwrap()) .collect(); let scorer: Box = box Intersection::from(scorers); return scorer; @@ -122,7 +121,7 @@ impl BooleanWeight { { let is_all_term_queries = scorers.iter().all(|scorer| { let scorer_ref: &Scorer = scorer.borrow(); - Downcast::>::is_type(scorer_ref) + Downcast::::is_type(scorer_ref) }); if is_all_term_queries { if scorers.len() == 2 { @@ -130,9 +129,9 @@ impl BooleanWeight { let left = scorers.pop().unwrap(); return box IntersectionTwoTerms::new(left, right); } else { - let mut scorers: Vec> = scorers + let scorers: Vec = scorers .into_iter() - .map(|scorer| *Downcast::>::downcast(scorer).unwrap()) + .map(|scorer| *Downcast::::downcast(scorer).unwrap()) .collect(); let scorer: Box = box Intersection::from(scorers); return scorer; diff --git a/src/query/boolean_query/mod.rs b/src/query/boolean_query/mod.rs index ca78939ad..24b8eb919 100644 --- a/src/query/boolean_query/mod.rs +++ b/src/query/boolean_query/mod.rs @@ -12,7 +12,6 @@ mod tests { use query::TermQuery; use query::Intersection; use query::Scorer; - use query::term_query::TermScorer; use collector::tests::TestCollector; use Index; use downcast::Downcast; @@ -20,8 +19,7 @@ mod tests { use query::QueryParser; use query::RequiredOptionalScorer; use query::score_combiner::SumWithCoordsCombiner; - - + use query::term_query::TermScorerNoDeletes; fn aux_test_helper() -> (Index, Field) { let mut schema_builder = SchemaBuilder::default(); @@ -73,7 +71,7 @@ mod tests { let searcher = index.searcher(); let weight = query.weight(&*searcher, true).unwrap(); let scorer = weight.scorer(searcher.segment_reader(0u32)).unwrap(); - assert!(Downcast::::is_type(&*scorer)); + assert!(Downcast::::is_type(&*scorer)); } #[test] @@ -85,7 +83,7 @@ mod tests { let query = query_parser.parse_query("+a +b +c").unwrap(); let weight = query.weight(&*searcher, true).unwrap(); let scorer = weight.scorer(searcher.segment_reader(0u32)).unwrap(); - assert!(Downcast::>>::is_type(&*scorer)); + assert!(Downcast::>::is_type(&*scorer)); } { let query = query_parser.parse_query("+a +(b c)").unwrap(); @@ -95,8 +93,6 @@ mod tests { } } - use postings::NoDelete; - #[test] pub fn test_boolean_reqopt() { let (index, text_field) = aux_test_helper(); @@ -112,7 +108,8 @@ mod tests { let query = query_parser.parse_query("+a b").unwrap(); let weight = query.weight(&*searcher, false).unwrap(); let scorer = weight.scorer(searcher.segment_reader(0u32)).unwrap(); - assert!(Downcast::>::is_type(&*scorer)); + println!("{:?}", scorer.type_name()); + assert!(Downcast::::is_type(&*scorer)); } } diff --git a/src/query/intersection_two.rs b/src/query/intersection_two.rs index 59a7a2366..1b2d7a1bf 100644 --- a/src/query/intersection_two.rs +++ b/src/query/intersection_two.rs @@ -2,9 +2,6 @@ use docset::DocSet; use query::Scorer; use DocId; use Score; -use std::mem; -use query::term_query::TermScorer; -use postings::DeleteSet; use SkipResult; @@ -15,7 +12,7 @@ pub struct IntersectionTwoTerms { } impl IntersectionTwoTerms { - pub fn new(left: TDocSet, mut right: TDocSet) -> IntersectionTwoTerms { + pub fn new(left: TDocSet, right: TDocSet) -> IntersectionTwoTerms { IntersectionTwoTerms { left, right diff --git a/src/query/term_query/mod.rs b/src/query/term_query/mod.rs index 11e70aa8a..cb9210a72 100644 --- a/src/query/term_query/mod.rs +++ b/src/query/term_query/mod.rs @@ -6,6 +6,13 @@ pub use self::term_query::TermQuery; pub use self::term_weight::TermWeight; pub use self::term_scorer::TermScorer; +use postings::SegmentPostings; +use postings::NoDelete; +use fastfield::DeleteBitSet; + +pub(crate) type TermScorerWithDeletes = TermScorer>; +pub(crate) type TermScorerNoDeletes = TermScorer>; + #[cfg(test)] mod tests { diff --git a/src/query/term_query/term_scorer.rs b/src/query/term_query/term_scorer.rs index 9bb0763e3..bdd6e65ac 100644 --- a/src/query/term_query/term_scorer.rs +++ b/src/query/term_query/term_scorer.rs @@ -1,25 +1,17 @@ use Score; use DocId; use docset::{DocSet, SkipResult}; -use postings::SegmentPostings; use query::Scorer; -use postings::Postings; use fastfield::FastFieldReader; -use postings::{NoDelete, DeleteSet}; +use postings::Postings; -pub struct TermScorer { +pub struct TermScorer { pub idf: Score, pub fieldnorm_reader_opt: Option>, - pub postings: SegmentPostings, + pub postings: TPostings, } -impl TermScorer { - pub fn postings(&self) -> &SegmentPostings { - &self.postings - } -} - -impl DocSet for TermScorer { +impl DocSet for TermScorer { fn advance(&mut self) -> bool { self.postings.advance() } @@ -37,7 +29,7 @@ impl DocSet for TermScorer { } } -impl Scorer for TermScorer { +impl Scorer for TermScorer { fn score(&mut self) -> Score { let doc = self.postings.doc(); let tf = match self.fieldnorm_reader_opt { diff --git a/src/query/term_query/term_weight.rs b/src/query/term_query/term_weight.rs index 2b2da02df..6353541f4 100644 --- a/src/query/term_query/term_weight.rs +++ b/src/query/term_query/term_weight.rs @@ -7,7 +7,7 @@ use postings::SegmentPostings; use schema::IndexRecordOption; use super::term_scorer::TermScorer; use fastfield::DeleteBitSet; -use postings::{DeleteSet, NoDelete}; +use postings::NoDelete; use Result; pub struct TermWeight { From c3fbc4c8fad942cd2c587220b0d9b731ebfaede6 Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Thu, 22 Feb 2018 10:43:06 +0900 Subject: [PATCH 03/24] Simplified a notch TinySet::pop_lowest() --- src/common/bitset.rs | 20 +++++--------------- src/query/bitset/mod.rs | 2 +- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/src/common/bitset.rs b/src/common/bitset.rs index c0edea292..08428590c 100644 --- a/src/common/bitset.rs +++ b/src/common/bitset.rs @@ -84,23 +84,12 @@ impl TinySet { /// and removes it. #[inline(always)] pub fn pop_lowest(&mut self) -> Option { - if let Some(lowest) = self.lowest() { - self.0 ^= TinySet::singleton(lowest).0; - Some(lowest) - } else { - None - } - } - - /// Returns the lowest element in the `TinySet` - /// (or None if the set is empty). - #[inline(always)] - pub fn lowest(&mut self) -> Option { if self.is_empty() { None } else { - let least_significant_bit = self.0.trailing_zeros() as u32; - Some(least_significant_bit) + let lowest = self.0.trailing_zeros() as u32; + self.0 ^= TinySet::singleton(lowest).0; + Some(lowest) } } @@ -366,7 +355,7 @@ mod tests { #[bench] fn bench_tinyset_pop(b: &mut test::Bencher) { - b.iter(|| test::black_box(TinySet::singleton(31u32)).pop_lowest()); + b.iter(|| test::black_box(TinySet(321u64)).pop_lowest()); } #[bench] @@ -387,4 +376,5 @@ mod tests { fn bench_bitset_initialize(b: &mut test::Bencher) { b.iter(|| BitSet::with_max_value(1_000_000)); } + } diff --git a/src/query/bitset/mod.rs b/src/query/bitset/mod.rs index aa8c4aed3..53947f16a 100644 --- a/src/query/bitset/mod.rs +++ b/src/query/bitset/mod.rs @@ -244,7 +244,7 @@ mod tests { #[bench] fn bench_bitset_1pct_clone_iterate(b: &mut test::Bencher) { - let els = tests::generate_nonunique_unsorted(1_000_000u32, 10_000); + let els = tests::sample(1_000_000u32, 0.01); let mut bitset = BitSet::with_max_value(1_000_000); for el in els { bitset.insert(el); From 1b94a3e382f9a01c1d6ca942b0d493f61cb8bbfe Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Thu, 22 Feb 2018 22:36:52 +0900 Subject: [PATCH 04/24] Phrase query optimisation --- src/postings/mod.rs | 2 +- src/postings/postings.rs | 7 +- src/postings/segment_postings.rs | 31 ++-- src/query/boolean_query/boolean_weight.rs | 56 +------ src/query/intersection.rs | 184 ++++++++++++++++------ src/query/mod.rs | 3 +- src/query/phrase_query/phrase_scorer.rs | 152 +++++++++--------- src/query/phrase_query/phrase_weight.rs | 33 ++-- 8 files changed, 270 insertions(+), 198 deletions(-) diff --git a/src/postings/mod.rs b/src/postings/mod.rs index 2296a4e3e..7734d1d6e 100644 --- a/src/postings/mod.rs +++ b/src/postings/mod.rs @@ -551,7 +551,7 @@ pub mod tests { .inverted_index(TERM_D.field()) .read_postings(&*TERM_D, IndexRecordOption::Basic) .unwrap(); - let mut intersection = Intersection::from(vec![ + let mut intersection = Intersection::new(vec![ segment_postings_a, segment_postings_b, segment_postings_c, diff --git a/src/postings/postings.rs b/src/postings/postings.rs index 92056b958..2e1a5c3e8 100644 --- a/src/postings/postings.rs +++ b/src/postings/postings.rs @@ -13,7 +13,12 @@ use docset::DocSet; pub trait Postings: DocSet + 'static { /// Returns the term frequency fn term_freq(&self) -> u32; + /// Returns the list of positions of the term, expressed as a list of /// token ordinals. - fn positions(&self) -> &[u32]; + fn positions_with_offset(&self, offset: u32) -> &[u32]; + + fn positions(&self) -> &[u32] { + self.positions_with_offset(0u32) + } } diff --git a/src/postings/segment_postings.rs b/src/postings/segment_postings.rs index 5531c944e..a67a5b0da 100644 --- a/src/postings/segment_postings.rs +++ b/src/postings/segment_postings.rs @@ -44,20 +44,27 @@ impl PositionComputer { ); } - pub fn positions(&mut self, term_freq: usize) -> &[u32] { + pub fn positions(&mut self, term_freq: usize, offset: u32) -> &[u32] { if let Some(num_skip) = self.position_to_skip { - self.positions.resize(term_freq, 0u32); - self.positions_stream.skip(num_skip); - self.positions_stream.read(&mut self.positions[..term_freq]); - - let mut cum = 0u32; - for i in 0..term_freq as usize { - cum += self.positions[i]; - self.positions[i] = cum; + let capacity = self.positions.capacity(); + if capacity < term_freq { + let extra_capacity = term_freq - self.positions.len(); + self.positions.reserve(extra_capacity); } + unsafe {self.positions.set_len(term_freq)}; + self.positions_stream.skip(num_skip); + let positions_buf = &mut self.positions[..term_freq]; + self.positions_stream.read(positions_buf); self.position_to_skip = None; + let mut cum = offset; + for position_mut in positions_buf.iter_mut() { + cum += *position_mut; + *position_mut = cum; + } + positions_buf + } else { + &self.positions[..term_freq] } - &self.positions[..term_freq] } } @@ -314,12 +321,12 @@ impl Postings for SegmentPostings { self.block_cursor.freq(self.cur) } - fn positions(&self) -> &[u32] { + fn positions_with_offset(&self, offset: u32) -> &[u32] { let term_freq = self.term_freq(); self.position_computer .as_ref() .map(|position_computer| unsafe { - (&mut *position_computer.get()).positions(term_freq as usize) + (&mut *position_computer.get()).positions(term_freq as usize, offset) }) .unwrap_or(&EMPTY_POSITIONS[..]) } diff --git a/src/query/boolean_query/boolean_weight.rs b/src/query/boolean_query/boolean_weight.rs index 51a3412f9..f5b2511a2 100644 --- a/src/query/boolean_query/boolean_weight.rs +++ b/src/query/boolean_query/boolean_weight.rs @@ -1,6 +1,6 @@ use query::Weight; use core::SegmentReader; -use query::{Intersection, Union}; +use query::Union; use std::collections::HashMap; use query::EmptyScorer; use query::Scorer; @@ -9,9 +9,9 @@ use std::borrow::Borrow; use query::Exclude; use query::Occur; use query::RequiredOptionalScorer; -use query::IntersectionTwoTerms; use query::score_combiner::{DoNothingCombiner, ScoreCombiner, SumWithCoordsCombiner}; use Result; +use query::intersect_scorers; use query::term_query::{TermScorerWithDeletes, TermScorerNoDeletes}; @@ -93,56 +93,8 @@ impl BooleanWeight { .map(scorer_union::); let must_scorer_opt: Option> = - per_occur_scorers.remove(&Occur::Must).map(|mut scorers| { - if scorers.len() == 1 { - return scorers.into_iter().next().unwrap(); - } - scorers.sort_by_key(|scorer| scorer.size_hint()); - { - let is_all_term_queries = scorers.iter().all(|scorer| { - let scorer_ref: &Scorer = scorer.borrow(); - Downcast::::is_type(scorer_ref) - }); - if is_all_term_queries { - if scorers.len() == 2 { - let right = scorers.pop().unwrap(); - let left = scorers.pop().unwrap(); - return box IntersectionTwoTerms::new(left, right); - } else { - let scorers: Vec = scorers - .into_iter() - .map(|scorer| *Downcast::::downcast(scorer).unwrap()) - .collect(); - let scorer: Box = box Intersection::from(scorers); - return scorer; - } - } - } - { - let is_all_term_queries = scorers.iter().all(|scorer| { - let scorer_ref: &Scorer = scorer.borrow(); - Downcast::::is_type(scorer_ref) - }); - if is_all_term_queries { - if scorers.len() == 2 { - let right = scorers.pop().unwrap(); - let left = scorers.pop().unwrap(); - return box IntersectionTwoTerms::new(left, right); - } else { - let scorers: Vec = scorers - .into_iter() - .map(|scorer| *Downcast::::downcast(scorer).unwrap()) - .collect(); - let scorer: Box = box Intersection::from(scorers); - return scorer; - } - } - } - { - let scorer: Box = box Intersection::from(scorers); - scorer - } - }); + per_occur_scorers.remove(&Occur::Must) + .map(intersect_scorers); let positive_scorer: Box = match (should_scorer_opt, must_scorer_opt) { (Some(should_scorer), Some(must_scorer)) => { diff --git a/src/query/intersection.rs b/src/query/intersection.rs index f2e132199..5635252ce 100644 --- a/src/query/intersection.rs +++ b/src/query/intersection.rs @@ -1,77 +1,170 @@ use docset::{DocSet, SkipResult}; use query::Scorer; +use query::EmptyScorer; use DocId; +use downcast::Downcast; +use std::borrow::Borrow; use Score; +use postings::Postings; +use query::term_query::{TermScorerNoDeletes, TermScorerWithDeletes}; + +pub fn intersect_scorers(mut docsets: Vec>) -> Box { + docsets.sort_by(|left, right| right.size_hint().cmp(&left.size_hint())); + let rarest_opt = docsets.pop(); + let second_rarest_opt = docsets.pop(); + docsets.reverse(); + match (rarest_opt, second_rarest_opt) { + (None, None) => box EmptyScorer, + (Some(single_docset), None) => single_docset, + (Some(left), Some(right)) => { + { + if [&left, &right].into_iter().all(|scorer| { + let scorer_ref: &Scorer = (*scorer).borrow(); + Downcast::::is_type(scorer_ref) + }) { + let left = *Downcast::::downcast(left).unwrap(); + let right = *Downcast::::downcast(right).unwrap(); + return box Intersection { + left, + right, + others: docsets, + } + } + } + { + if [&left, &right].into_iter() + .all(|scorer| { + let scorer_ref: &Scorer = (*scorer).borrow(); + Downcast::::is_type(scorer_ref) + }) { + let left = *Downcast::::downcast(left).unwrap(); + let right = *Downcast::::downcast(right).unwrap(); + return box Intersection { + left, + right, + others: docsets, + } + } + } + { + return box Intersection { + left, + right, + others: docsets + } + } + } + _ => { unreachable!(); } + } +} /// Creates a `DocSet` that iterator through the intersection of two `DocSet`s. -pub struct Intersection { - docsets: Vec, - finished: bool, - doc: DocId, +pub struct Intersection> { + left: TDocSet, + right: TDocSet, + others: Vec } -impl From> for Intersection { - fn from(mut docsets: Vec) -> Intersection { +impl Intersection { + pub(crate) fn new(mut docsets: Vec) -> Intersection { assert!(docsets.len() >= 2); - docsets.sort_by_key(|docset| docset.size_hint()); + docsets.sort_by(|left, right| right.size_hint().cmp(&left.size_hint())); + let left = docsets.pop().unwrap(); + let right = docsets.pop().unwrap(); + docsets.reverse(); Intersection { - docsets, - finished: false, - doc: 0u32, + left, + right, + others: docsets } } } -impl Intersection { - /// Returns an array to the underlying `DocSet`s of the intersection. - /// These `DocSet` are in the same position as the `IntersectionDocSet`, - /// so that user can access their `docfreq` and `positions`. - pub fn docsets(&self) -> &[TDocSet] { - &self.docsets[..] +impl Intersection { + pub fn docset(&self, ord: usize) -> &TDocSet { + match ord { + 0 => &self.left, + 1 => &self.right, + n => &self.others[n - 2] + } } } -impl DocSet for Intersection { +impl DocSet for Intersection { #[allow(never_loop)] fn advance(&mut self) -> bool { - if self.finished { + let (left, right) = (&mut self.left, &mut self.right); + if !left.advance() { return false; } - - let mut candidate_doc = self.doc; - let mut candidate_ord = self.docsets.len(); + let mut candidate = left.doc(); + let mut other_candidate_ord: usize = usize::max_value(); 'outer: loop { - for (ord, docset) in self.docsets.iter_mut().enumerate() { - if ord != candidate_ord { + // In the first part we look for a document in the intersection + // of the two rarest `DocSet` in the intersection. + loop { + match right.skip_next(candidate) { + SkipResult::Reached => { break; } + SkipResult::OverStep => { + candidate = right.doc(); + other_candidate_ord = usize::max_value(); + } + SkipResult::End => { return false; } + } + match left.skip_next(candidate) { + SkipResult::Reached => { break; } + SkipResult::OverStep => { + candidate = left.doc(); + other_candidate_ord = usize::max_value(); + } + SkipResult::End => { return false; } + } + } + + // test the remaining scorers; + for (ord, docset) in self.others.iter_mut().enumerate() { + if ord != other_candidate_ord { // `candidate_ord` is already at the // right position. // // Calling `skip_next` would advance this docset // and miss it. - match docset.skip_next(candidate_doc) { + match docset.skip_next(candidate) { SkipResult::Reached => {} SkipResult::OverStep => { // this is not in the intersection, // let's update our candidate. - candidate_doc = docset.doc(); - candidate_ord = ord; + candidate = docset.doc(); + match left.skip_next(candidate) { + SkipResult::Reached => { + other_candidate_ord = ord; + } + SkipResult::OverStep => { + other_candidate_ord = usize::max_value(); + } + SkipResult::End => { + return false; + } + } continue 'outer; } SkipResult::End => { - self.finished = true; return false; } } } } - self.doc = candidate_doc; return true; } } fn skip_next(&mut self, target: DocId) -> SkipResult { + + unimplemented!("werwer"); + + /* // We optimize skipping by skipping every single member // of the intersection to target. let mut current_target: DocId = target; @@ -96,8 +189,6 @@ impl DocSet for Intersection { SkipResult::Reached => {} } } - - self.doc = current_target; if target == current_target { return SkipResult::Reached; } else { @@ -105,27 +196,22 @@ impl DocSet for Intersection { return SkipResult::OverStep; } } + */ } fn doc(&self) -> DocId { - self.doc + self.left.doc() } fn size_hint(&self) -> u32 { - self.docsets - .iter() - .map(|docset| docset.size_hint()) - .min() - .unwrap_or(0u32) + self.left.size_hint() } } -impl Scorer for Intersection -where - TScorer: Scorer, -{ +impl Scorer for Intersection +where TScorer: Scorer, TOtherScorer: Scorer { fn score(&mut self) -> Score { - self.docsets.iter_mut().map(Scorer::score).sum() + self.left.score() + self.right.score() + self.others.iter_mut().map(Scorer::score).sum::() } } @@ -141,7 +227,7 @@ mod tests { { let left = VecDocSet::from(vec![1, 3, 9]); let right = VecDocSet::from(vec![3, 4, 9, 18]); - let mut intersection = Intersection::from(vec![left, right]); + let mut intersection = Intersection::new(vec![left, right]); assert!(intersection.advance()); assert_eq!(intersection.doc(), 3); assert!(intersection.advance()); @@ -152,7 +238,7 @@ mod tests { let a = VecDocSet::from(vec![1, 3, 9]); let b = VecDocSet::from(vec![3, 4, 9, 18]); let c = VecDocSet::from(vec![1, 5, 9, 111]); - let mut intersection = Intersection::from(vec![a, b, c]); + let mut intersection = Intersection::new(vec![a, b, c]); assert!(intersection.advance()); assert_eq!(intersection.doc(), 9); assert!(!intersection.advance()); @@ -163,7 +249,7 @@ mod tests { fn test_intersection_zero() { let left = VecDocSet::from(vec![0]); let right = VecDocSet::from(vec![0]); - let mut intersection = Intersection::from(vec![left, right]); + let mut intersection = Intersection::new(vec![left, right]); assert!(intersection.advance()); assert_eq!(intersection.doc(), 0); } @@ -172,7 +258,7 @@ mod tests { fn test_intersection_skip() { let left = VecDocSet::from(vec![0, 1, 2, 4]); let right = VecDocSet::from(vec![2, 5]); - let mut intersection = Intersection::from(vec![left, right]); + let mut intersection = Intersection::new(vec![left, right]); assert_eq!(intersection.skip_next(2), SkipResult::Reached); assert_eq!(intersection.doc(), 2); } @@ -183,7 +269,7 @@ mod tests { || { let left = VecDocSet::from(vec![4]); let right = VecDocSet::from(vec![2, 5]); - box Intersection::from(vec![left, right]) + box Intersection::new(vec![left, right]) }, vec![0, 2, 4, 5, 6], ); @@ -193,13 +279,13 @@ mod tests { let mut right = VecDocSet::from(vec![2, 5, 10]); left.advance(); right.advance(); - box Intersection::from(vec![left, right]) + box Intersection::new(vec![left, right]) }, vec![0, 1, 2, 3, 4, 5, 6, 7, 10, 11], ); test_skip_against_unoptimized( || { - box Intersection::from(vec![ + box Intersection::new(vec![ VecDocSet::from(vec![1, 4, 5, 6]), VecDocSet::from(vec![1, 2, 5, 6]), VecDocSet::from(vec![1, 4, 5, 6]), @@ -216,7 +302,7 @@ mod tests { let a = VecDocSet::from(vec![1, 3]); let b = VecDocSet::from(vec![1, 4]); let c = VecDocSet::from(vec![3, 9]); - let mut intersection = Intersection::from(vec![a, b, c]); + let mut intersection = Intersection::new(vec![a, b, c]); assert!(!intersection.advance()); } } diff --git a/src/query/mod.rs b/src/query/mod.rs index df68004c2..b3ed49591 100644 --- a/src/query/mod.rs +++ b/src/query/mod.rs @@ -16,7 +16,6 @@ mod range_query; mod exclude; mod union; mod intersection; -mod intersection_two; mod reqopt_scorer; #[cfg(test)] @@ -46,4 +45,4 @@ pub use self::weight::Weight; pub use self::all_query::{AllQuery, AllScorer, AllWeight}; pub use self::range_query::RangeQuery; pub use self::scorer::ConstScorer; -pub use self::intersection_two::IntersectionTwoTerms; +pub use self::intersection::intersect_scorers; diff --git a/src/query/phrase_query/phrase_scorer.rs b/src/query/phrase_query/phrase_scorer.rs index a58dda6a5..334b7fdc0 100644 --- a/src/query/phrase_query/phrase_scorer.rs +++ b/src/query/phrase_query/phrase_scorer.rs @@ -1,116 +1,126 @@ use DocId; use docset::{DocSet, SkipResult}; -use postings::{Postings, SegmentPostings}; +use postings::Postings; use query::{Intersection, Scorer}; use fastfield::DeleteBitSet; +use query::intersect_scorers; +use Score; +use std::cmp::Ordering; +use std::mem; -struct PostingsWithOffset { + +struct PostingsWithOffset { offset: u32, - segment_postings: SegmentPostings, + postings: TPostings } -impl PostingsWithOffset { - pub fn new(segment_postings: SegmentPostings, offset: u32) -> PostingsWithOffset { +impl PostingsWithOffset { + pub fn new(segment_postings: TPostings, offset: u32) -> PostingsWithOffset { PostingsWithOffset { offset, - segment_postings, + postings: segment_postings } } -} -impl Postings for PostingsWithOffset { - fn term_freq(&self) -> u32 { - self.segment_postings.term_freq() - } - - fn positions(&self) -> &[u32] { - self.segment_postings.positions() + pub fn positions(&self) -> &[u32] { + self.postings.positions_with_offset(self.offset) } } -impl DocSet for PostingsWithOffset { +impl DocSet for PostingsWithOffset { fn advance(&mut self) -> bool { - self.segment_postings.advance() + self.postings.advance() } fn skip_next(&mut self, target: DocId) -> SkipResult { - self.segment_postings.skip_next(target) + self.postings.skip_next(target) } fn doc(&self) -> DocId { - self.segment_postings.doc() + self.postings.doc() } fn size_hint(&self) -> u32 { - self.segment_postings.size_hint() + self.postings.size_hint() } } -pub struct PhraseScorer { - intersection_docset: Intersection, +pub struct PhraseScorer { + intersection_docset: Intersection, PostingsWithOffset>, + num_docsets: usize, + source: Vec, + result: Vec } -impl PhraseScorer { - pub fn new(term_postings: Vec>) -> PhraseScorer { - let postings_with_offsets: Vec<_> = term_postings +#[inline(always)] +fn intersection_arr(left: &[u32], right: &[u32], output: &mut [u32]) -> usize { + let mut left_i = 0; + let mut right_i = 0; + let mut count = 0; + while left_i < left.len() && right_i < right.len() { + if left[left_i] < right[right_i] { + left_i += 1; + } else if right[right_i] < left[left_i] { + right_i += 1; + } else { + output[count] = left[left_i]; + count+=1; + left_i += 1; + right_i += 1; + } + } + count +} + +impl PhraseScorer { + + pub fn new(term_postings: Vec) -> PhraseScorer { + let num_docsets = term_postings.len(); + let postings_with_offsets = term_postings .into_iter() .enumerate() - .map(|(offset, postings)| PostingsWithOffset::new(postings, offset as u32)) - .collect(); + .map(|(offset, postings)| PostingsWithOffset::new(postings, (num_docsets - offset) as u32)) + .collect::>(); PhraseScorer { - intersection_docset: Intersection::from(postings_with_offsets), + intersection_docset: Intersection::new(postings_with_offsets), + num_docsets, + source: Vec::with_capacity(100), + result: Vec::with_capacity(100) } } - fn phrase_match(&self) -> bool { - // TODO maybe we could avoid decoding positions lazily for all terms - // when there is > 2 terms. - // - // For instance for the query "A B C", the position of "C" do not need - // to be decoded if "A B" had no match. - let docsets = self.intersection_docset.docsets(); - let mut positions_arr: Vec<&[u32]> = vec![&[]; docsets.len()]; - for docset in docsets { - positions_arr[docset.offset as usize] = docset.positions(); - } - - let num_postings = positions_arr.len() as u32; - - let mut ord = 1u32; - let mut pos_candidate = positions_arr[0][0]; - positions_arr[0] = &(positions_arr[0])[1..]; - let mut count_matching = 1; - - #[cfg_attr(feature = "cargo-clippy", allow(never_loop))] - 'outer: loop { - let target = pos_candidate + ord; - let positions = positions_arr[ord as usize]; - for (i, pos_i) in positions.iter().cloned().enumerate() { - if pos_i < target { - continue; - } - if pos_i == target { - count_matching += 1; - if count_matching == num_postings { - return true; - } - } else if pos_i > target { - count_matching = 1; - pos_candidate = positions[i] - ord; - positions_arr[ord as usize] = &(positions_arr[ord as usize])[(i + 1)..]; - } - ord += 1; - if ord == num_postings { - ord = 0; - } - continue 'outer; + fn phrase_match(&mut self) -> bool { + // TODO early exit when we don't care about th phrase frequency + let mut intersection_len; + { + let left = self.intersection_docset.docset(0).positions(); + let right = self.intersection_docset.docset(1).positions(); + let max_intersection_len = left.len().min(right.len()); + if max_intersection_len > self.result.len() { + self.result.resize(max_intersection_len, 0u32); + self.source.resize(max_intersection_len, 0u32) } + intersection_len = intersection_arr(left, right, &mut self.result[..]); + } + if intersection_len == 0 { return false; } + for i in 2..self.num_docsets { + mem::swap(&mut self.source, &mut self.result); + let term_positions = self.intersection_docset.docset(i).positions(); + intersection_len = intersection_arr( + &self.source[..intersection_len], + term_positions, + &mut self.result[..]); + if intersection_len == 0 { + return false; + } + } + return true; } } -impl DocSet for PhraseScorer { +impl DocSet for PhraseScorer { fn advance(&mut self) -> bool { while self.intersection_docset.advance() { if self.phrase_match() { @@ -147,7 +157,7 @@ impl DocSet for PhraseScorer { } } -impl Scorer for PhraseScorer { +impl Scorer for PhraseScorer { fn score(&mut self) -> f32 { 1f32 } diff --git a/src/query/phrase_query/phrase_weight.rs b/src/query/phrase_query/phrase_weight.rs index 84825b15c..026a9c838 100644 --- a/src/query/phrase_query/phrase_weight.rs +++ b/src/query/phrase_query/phrase_weight.rs @@ -26,17 +26,30 @@ impl PhraseWeight { impl Weight for PhraseWeight { fn scorer(&self, reader: &SegmentReader) -> Result> { - let mut term_postings_list = Vec::new(); - for term in &self.phrase_terms { - if let Some(postings) = reader - .inverted_index(term.field()) - .read_postings(term, IndexRecordOption::WithFreqsAndPositions) - { - term_postings_list.push(postings); - } else { - return Ok(box EmptyScorer); + if reader.has_deletes() { + let mut term_postings_list = Vec::new(); + for term in &self.phrase_terms { + if let Some(postings) = reader + .inverted_index(term.field()) + .read_postings(term, IndexRecordOption::WithFreqsAndPositions) { + term_postings_list.push(postings); + } else { + return Ok(box EmptyScorer); + } } + Ok(box PhraseScorer::new(term_postings_list)) + } else { + let mut term_postings_list = Vec::new(); + for term in &self.phrase_terms { + if let Some(postings) = reader + .inverted_index(term.field()) + .read_postings_no_deletes(term, IndexRecordOption::WithFreqsAndPositions) { + term_postings_list.push(postings); + } else { + return Ok(box EmptyScorer); + } + } + Ok(box PhraseScorer::new(term_postings_list)) } - Ok(box PhraseScorer::new(term_postings_list)) } } From be830b03c51ddacb9dc409d7ef04b749372adb7a Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Fri, 23 Feb 2018 11:55:23 +0900 Subject: [PATCH 05/24] Bugfix in intersection.advance and impl skip_next --- src/query/intersection.rs | 55 +++++++++++++++---------- src/query/phrase_query/phrase_scorer.rs | 5 --- 2 files changed, 33 insertions(+), 27 deletions(-) diff --git a/src/query/intersection.rs b/src/query/intersection.rs index 5635252ce..4a60fcb01 100644 --- a/src/query/intersection.rs +++ b/src/query/intersection.rs @@ -5,10 +5,10 @@ use DocId; use downcast::Downcast; use std::borrow::Borrow; use Score; -use postings::Postings; use query::term_query::{TermScorerNoDeletes, TermScorerWithDeletes}; pub fn intersect_scorers(mut docsets: Vec>) -> Box { + let num_docsets = docsets.len(); docsets.sort_by(|left, right| right.size_hint().cmp(&left.size_hint())); let rarest_opt = docsets.pop(); let second_rarest_opt = docsets.pop(); @@ -28,6 +28,7 @@ pub fn intersect_scorers(mut docsets: Vec>) -> Box { left, right, others: docsets, + num_docsets } } } @@ -43,6 +44,7 @@ pub fn intersect_scorers(mut docsets: Vec>) -> Box { left, right, others: docsets, + num_docsets } } } @@ -50,7 +52,8 @@ pub fn intersect_scorers(mut docsets: Vec>) -> Box { return box Intersection { left, right, - others: docsets + others: docsets, + num_docsets } } } @@ -62,12 +65,14 @@ pub fn intersect_scorers(mut docsets: Vec>) -> Box { pub struct Intersection> { left: TDocSet, right: TDocSet, - others: Vec + others: Vec, + num_docsets: usize } impl Intersection { pub(crate) fn new(mut docsets: Vec) -> Intersection { - assert!(docsets.len() >= 2); + let num_docsets = docsets.len(); + assert!(num_docsets >= 2); docsets.sort_by(|left, right| right.size_hint().cmp(&left.size_hint())); let left = docsets.pop().unwrap(); let right = docsets.pop().unwrap(); @@ -75,7 +80,8 @@ impl Intersection { Intersection { left, right, - others: docsets + others: docsets, + num_docsets } } } @@ -90,13 +96,25 @@ impl Intersection { } } +impl Intersection { + pub fn docset_mut(&mut self, ord: usize) -> &mut DocSet { + match ord { + 0 => &mut self.left, + 1 => &mut self.right, + n => &mut self.others[n - 2] + } + } +} + impl DocSet for Intersection { #[allow(never_loop)] fn advance(&mut self) -> bool { let (left, right) = (&mut self.left, &mut self.right); + if !left.advance() { return false; } + let mut candidate = left.doc(); let mut other_candidate_ord: usize = usize::max_value(); @@ -112,6 +130,7 @@ impl DocSet for Intersection { return false; } } + match left.skip_next(candidate) { SkipResult::Reached => { break; } SkipResult::OverStep => { @@ -120,8 +139,8 @@ impl DocSet for Intersection { return false; } } - } + } // test the remaining scorers; for (ord, docset) in self.others.iter_mut().enumerate() { if ord != other_candidate_ord { @@ -137,41 +156,34 @@ impl DocSet for Intersection { - other_candidate_ord = ord; - } + SkipResult::Reached => { other_candidate_ord = ord; } SkipResult::OverStep => { + candidate = left.doc(); other_candidate_ord = usize::max_value(); } - SkipResult::End => { - return false; - } + SkipResult::End => { return false; } } continue 'outer; } - SkipResult::End => { - return false; - } + SkipResult::End => { return false; } } } } - return true; } } + fn skip_next(&mut self, target: DocId) -> SkipResult { - unimplemented!("werwer"); - - /* // We optimize skipping by skipping every single member // of the intersection to target. let mut current_target: DocId = target; - let mut current_ord = self.docsets.len(); + let mut current_ord = self.num_docsets; 'outer: loop { - for (ord, docset) in self.docsets.iter_mut().enumerate() { + for ord in 0..self.num_docsets { + let docset = self.docset_mut(ord); if ord == current_ord { continue; } @@ -196,7 +208,6 @@ impl DocSet for Intersection DocId { diff --git a/src/query/phrase_query/phrase_scorer.rs b/src/query/phrase_query/phrase_scorer.rs index 334b7fdc0..431b77abc 100644 --- a/src/query/phrase_query/phrase_scorer.rs +++ b/src/query/phrase_query/phrase_scorer.rs @@ -2,10 +2,6 @@ use DocId; use docset::{DocSet, SkipResult}; use postings::Postings; use query::{Intersection, Scorer}; -use fastfield::DeleteBitSet; -use query::intersect_scorers; -use Score; -use std::cmp::Ordering; use std::mem; @@ -52,7 +48,6 @@ pub struct PhraseScorer { result: Vec } -#[inline(always)] fn intersection_arr(left: &[u32], right: &[u32], output: &mut [u32]) -> usize { let mut left_i = 0; let mut right_i = 0; From 23387b0ed05df6e8bbe7037835b738e2db81ada6 Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Sat, 24 Feb 2018 11:14:45 +0900 Subject: [PATCH 06/24] Positions writes to an external Vec --- src/indexer/merger.rs | 17 +++--- src/postings/postings.rs | 6 +-- src/postings/segment_postings.rs | 46 ++++++++-------- src/query/phrase_query/phrase_scorer.rs | 71 +++++++++++++++---------- 4 files changed, 74 insertions(+), 66 deletions(-) diff --git a/src/indexer/merger.rs b/src/indexer/merger.rs index 57f03a80a..bbdf7c2f7 100644 --- a/src/indexer/merger.rs +++ b/src/indexer/merger.rs @@ -314,14 +314,15 @@ impl IndexMerger { { // we make sure to only write the term iff // there is at least one document. - let positions: &[u32] = segment_postings.positions(); - let term_freq = segment_postings.term_freq(); - let delta_positions = delta_computer.compute_delta(positions); - field_serializer.write_doc( - remapped_doc_id, - term_freq, - delta_positions, - )?; + unreachable!(); +// let positions: &[u32] = segment_postings.positions(); +// let term_freq = segment_postings.term_freq(); +// let delta_positions = delta_computer.compute_delta(positions); +// field_serializer.write_doc( +// remapped_doc_id, +// term_freq, +// delta_positions, +// )?; } if !segment_postings.advance() { break; diff --git a/src/postings/postings.rs b/src/postings/postings.rs index 2e1a5c3e8..ca853efd5 100644 --- a/src/postings/postings.rs +++ b/src/postings/postings.rs @@ -16,9 +16,5 @@ pub trait Postings: DocSet + 'static { /// Returns the list of positions of the term, expressed as a list of /// token ordinals. - fn positions_with_offset(&self, offset: u32) -> &[u32]; - - fn positions(&self) -> &[u32] { - self.positions_with_offset(0u32) - } + fn positions_with_offset(&self, offset: u32, output: &mut Vec); } diff --git a/src/postings/segment_postings.rs b/src/postings/segment_postings.rs index a67a5b0da..3577ed2b4 100644 --- a/src/postings/segment_postings.rs +++ b/src/postings/segment_postings.rs @@ -23,7 +23,6 @@ struct PositionComputer { // if none, position are already loaded in // the positions vec. position_to_skip: Option, - positions: Vec, positions_stream: CompressedIntStream, } @@ -31,7 +30,6 @@ impl PositionComputer { pub fn new(positions_stream: CompressedIntStream) -> PositionComputer { PositionComputer { position_to_skip: None, - positions: vec![], positions_stream, } } @@ -44,26 +42,19 @@ impl PositionComputer { ); } - pub fn positions(&mut self, term_freq: usize, offset: u32) -> &[u32] { + pub fn positions(&mut self, offset: u32, output: &mut [u32]) { + let term_freq = output.len(); if let Some(num_skip) = self.position_to_skip { - let capacity = self.positions.capacity(); - if capacity < term_freq { - let extra_capacity = term_freq - self.positions.len(); - self.positions.reserve(extra_capacity); - } - unsafe {self.positions.set_len(term_freq)}; self.positions_stream.skip(num_skip); - let positions_buf = &mut self.positions[..term_freq]; - self.positions_stream.read(positions_buf); + self.positions_stream.read(output); self.position_to_skip = None; let mut cum = offset; - for position_mut in positions_buf.iter_mut() { - cum += *position_mut; - *position_mut = cum; + for output_mut in output.iter_mut() { + cum += *output_mut; + *output_mut = cum; } - positions_buf } else { - &self.positions[..term_freq] + panic!("Failed positions"); } } } @@ -321,14 +312,21 @@ impl Postings for SegmentPostings { self.block_cursor.freq(self.cur) } - fn positions_with_offset(&self, offset: u32) -> &[u32] { - let term_freq = self.term_freq(); - self.position_computer - .as_ref() - .map(|position_computer| unsafe { - (&mut *position_computer.get()).positions(term_freq as usize, offset) - }) - .unwrap_or(&EMPTY_POSITIONS[..]) + fn positions_with_offset(&self, offset: u32, output: &mut Vec) { + if let Some(ref position_computer) = self.position_computer.as_ref() { + let prev_capacity = output.capacity(); + let term_freq = self.term_freq() as usize; + if term_freq > prev_capacity { + let additional_len = term_freq - output.len(); + output.reserve(additional_len); + } + unsafe { + output.set_len(term_freq); + (&mut *position_computer.get()).positions(offset, &mut output[..]) + } + } else { + unimplemented!("You may not read positions twice!"); + } } } diff --git a/src/query/phrase_query/phrase_scorer.rs b/src/query/phrase_query/phrase_scorer.rs index 431b77abc..4accd1fb3 100644 --- a/src/query/phrase_query/phrase_scorer.rs +++ b/src/query/phrase_query/phrase_scorer.rs @@ -18,8 +18,8 @@ impl PostingsWithOffset { } } - pub fn positions(&self) -> &[u32] { - self.postings.positions_with_offset(self.offset) + pub fn positions(&self, output: &mut Vec) { + self.postings.positions_with_offset(self.offset, output) } } @@ -44,11 +44,11 @@ impl DocSet for PostingsWithOffset { pub struct PhraseScorer { intersection_docset: Intersection, PostingsWithOffset>, num_docsets: usize, - source: Vec, - result: Vec + left: Vec, + right: Vec } -fn intersection_arr(left: &[u32], right: &[u32], output: &mut [u32]) -> usize { +fn intersection_arr(left: &mut [u32], right: &[u32]) -> usize { let mut left_i = 0; let mut right_i = 0; let mut count = 0; @@ -58,7 +58,7 @@ fn intersection_arr(left: &[u32], right: &[u32], output: &mut [u32]) -> usize { } else if right[right_i] < left[left_i] { right_i += 1; } else { - output[count] = left[left_i]; + left[count] = left[left_i]; count+=1; left_i += 1; right_i += 1; @@ -67,6 +67,7 @@ fn intersection_arr(left: &[u32], right: &[u32], output: &mut [u32]) -> usize { count } + impl PhraseScorer { pub fn new(term_postings: Vec) -> PhraseScorer { @@ -79,34 +80,18 @@ impl PhraseScorer { PhraseScorer { intersection_docset: Intersection::new(postings_with_offsets), num_docsets, - source: Vec::with_capacity(100), - result: Vec::with_capacity(100) + left: Vec::with_capacity(100), + right: Vec::with_capacity(100) } } fn phrase_match(&mut self) -> bool { // TODO early exit when we don't care about th phrase frequency - let mut intersection_len; - { - let left = self.intersection_docset.docset(0).positions(); - let right = self.intersection_docset.docset(1).positions(); - let max_intersection_len = left.len().min(right.len()); - if max_intersection_len > self.result.len() { - self.result.resize(max_intersection_len, 0u32); - self.source.resize(max_intersection_len, 0u32) - } - intersection_len = intersection_arr(left, right, &mut self.result[..]); - } - if intersection_len == 0 { - return false; - } - for i in 2..self.num_docsets { - mem::swap(&mut self.source, &mut self.result); - let term_positions = self.intersection_docset.docset(i).positions(); - intersection_len = intersection_arr( - &self.source[..intersection_len], - term_positions, - &mut self.result[..]); + self.intersection_docset.docset(0).positions(&mut self.left); + let mut intersection_len = self.left.len(); + for i in 1..self.num_docsets { + self.intersection_docset.docset(i).positions(&mut self.right); + intersection_len = intersection_arr(&mut self.left[..intersection_len], &self.right[..]); if intersection_len == 0 { return false; } @@ -157,3 +142,31 @@ impl Scorer for PhraseScorer { 1f32 } } + +#[cfg(test)] +mod tests { + + use tests; + use test::Bencher; + use super::{intersection_arr, intersection_avx}; + + #[bench] + fn bench_intersection(b: &mut Bencher) { + let left = tests::sample_with_seed(100_000, 0.1, 1); + let right = tests::sample_with_seed(200_000, 0.05, 2); + let mut output = vec![0u32; 200_000]; + b.iter(|| { + intersection_arr(&left, &right, &mut output); + }); + } + + #[bench] + fn bench_intersection_avx(b: &mut Bencher) { + let left = tests::sample_with_seed(100_000, 0.1, 1); + let right = tests::sample_with_seed(200_000, 0.05, 2); + let mut output = vec![0u32; 200_000]; + b.iter(|| { + intersection_avx(&left, &right, &mut output); + }); + } +} \ No newline at end of file From 2c207598292a49b9e2df5ae2a4ccce58fff05e73 Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Sat, 24 Feb 2018 12:07:55 +0900 Subject: [PATCH 07/24] removed unsafecell for position computer --- src/postings/postings.rs | 2 +- src/postings/segment_postings.rs | 59 ++++++++++++------------- src/query/intersection.rs | 9 ++-- src/query/phrase_query/phrase_scorer.rs | 12 +++-- 4 files changed, 42 insertions(+), 40 deletions(-) diff --git a/src/postings/postings.rs b/src/postings/postings.rs index ca853efd5..f66c6434d 100644 --- a/src/postings/postings.rs +++ b/src/postings/postings.rs @@ -16,5 +16,5 @@ pub trait Postings: DocSet + 'static { /// Returns the list of positions of the term, expressed as a list of /// token ordinals. - fn positions_with_offset(&self, offset: u32, output: &mut Vec); + fn positions_with_offset(&mut self, offset: u32, output: &mut Vec); } diff --git a/src/postings/segment_postings.rs b/src/postings/segment_postings.rs index 3577ed2b4..410d67e27 100644 --- a/src/postings/segment_postings.rs +++ b/src/postings/segment_postings.rs @@ -35,11 +35,9 @@ impl PositionComputer { } pub fn add_skip(&mut self, num_skip: usize) { - self.position_to_skip = Some( - self.position_to_skip - .map(|prev_skip| prev_skip + num_skip) - .unwrap_or(0), - ); + self.position_to_skip = self.position_to_skip + .map(|prev_skip| prev_skip + num_skip) + .or(Some(0)); } pub fn positions(&mut self, offset: u32, output: &mut [u32]) { @@ -68,7 +66,7 @@ pub struct SegmentPostings { block_cursor: BlockSegmentPostings, cur: usize, delete_bitset: TDeleteSet, - position_computer: Option>, + position_computer: Option, } impl SegmentPostings { @@ -111,14 +109,7 @@ impl SegmentPostings { } impl SegmentPostings { - fn position_add_skip usize>(&self, num_skips_fn: F) { - if let Some(position_computer) = self.position_computer.as_ref() { - let num_skips = num_skips_fn(); - unsafe { - (*position_computer.get()).add_skip(num_skips); - } - } - } + /// Reads a Segment postings from an &[u8] @@ -132,13 +123,11 @@ impl SegmentPostings { delete_bitset: TDeleteSet, positions_stream_opt: Option, ) -> SegmentPostings { - let position_computer = - positions_stream_opt.map(|stream| UnsafeCell::new(PositionComputer::new(stream))); SegmentPostings { block_cursor: segment_block_postings, cur: COMPRESSION_BLOCK_SIZE, // cursor within the block delete_bitset, - position_computer, + position_computer: positions_stream_opt.map(PositionComputer::new), } } } @@ -149,7 +138,12 @@ impl DocSet for SegmentPostings { #[inline] fn advance(&mut self) -> bool { loop { - self.position_add_skip(|| self.term_freq() as usize); + { + if self.position_computer.is_some() { + let term_freq = self.term_freq() as usize; + self.position_computer.as_mut().unwrap().add_skip(term_freq); + } + } self.cur += 1; if self.cur >= self.block_cursor.block_len() { self.cur = 0; @@ -164,6 +158,7 @@ impl DocSet for SegmentPostings { } } + fn skip_next(&mut self, target: DocId) -> SkipResult { if !self.advance() { return SkipResult::End; @@ -185,17 +180,16 @@ impl DocSet for SegmentPostings { // so that position_add_skip will decide if it should // just set itself to Some(0) or effectively // add the term freq. - //let num_skips: u32 = ; - self.position_add_skip(|| { + if self.position_computer.is_some() { let freqs_skipped = &self.block_cursor.freqs()[self.cur..]; - let sum_freq: u32 = freqs_skipped.iter().cloned().sum(); - sum_freq as usize - }); - + let sum_freq: u32 = freqs_skipped.iter().sum() + self.position_computer.as_mut() + .unwrap() + .add_skip(sum_freq as usize); + } if !self.block_cursor.advance() { return SkipResult::End; } - self.cur = 0; } else { if target < current_doc { @@ -246,11 +240,13 @@ impl DocSet for SegmentPostings { // `doc` is now >= `target` let doc = block_docs[start]; - self.position_add_skip(|| { + if self.position_computer.is_some() { let freqs_skipped = &self.block_cursor.freqs()[self.cur..start]; let sum_freqs: u32 = freqs_skipped.iter().sum(); - sum_freqs as usize - }); + self.position_computer.as_mut() + .unwrap() + .add_skip(sum_freqs as usize); + } self.cur = start; @@ -312,8 +308,8 @@ impl Postings for SegmentPostings { self.block_cursor.freq(self.cur) } - fn positions_with_offset(&self, offset: u32, output: &mut Vec) { - if let Some(ref position_computer) = self.position_computer.as_ref() { + fn positions_with_offset(&mut self, offset: u32, output: &mut Vec) { + if self.position_computer.is_some() { let prev_capacity = output.capacity(); let term_freq = self.term_freq() as usize; if term_freq > prev_capacity { @@ -322,7 +318,7 @@ impl Postings for SegmentPostings { } unsafe { output.set_len(term_freq); - (&mut *position_computer.get()).positions(offset, &mut output[..]) + self.position_computer.as_mut().unwrap().positions(offset, &mut output[..]) } } else { unimplemented!("You may not read positions twice!"); @@ -608,3 +604,4 @@ mod tests { assert_eq!(block_segments.docs(), &[1, 3, 5]); } } + diff --git a/src/query/intersection.rs b/src/query/intersection.rs index 4a60fcb01..75598b237 100644 --- a/src/query/intersection.rs +++ b/src/query/intersection.rs @@ -86,12 +86,13 @@ impl Intersection { } } + impl Intersection { - pub fn docset(&self, ord: usize) -> &TDocSet { + pub fn docset_mut_specialized(&mut self, ord: usize) -> &mut TDocSet { match ord { - 0 => &self.left, - 1 => &self.right, - n => &self.others[n - 2] + 0 => &mut self.left, + 1 => &mut self.right, + n => &mut self.others[n - 2] } } } diff --git a/src/query/phrase_query/phrase_scorer.rs b/src/query/phrase_query/phrase_scorer.rs index 4accd1fb3..f77c63d68 100644 --- a/src/query/phrase_query/phrase_scorer.rs +++ b/src/query/phrase_query/phrase_scorer.rs @@ -18,7 +18,7 @@ impl PostingsWithOffset { } } - pub fn positions(&self, output: &mut Vec) { + pub fn positions(&mut self, output: &mut Vec) { self.postings.positions_with_offset(self.offset, output) } } @@ -86,11 +86,15 @@ impl PhraseScorer { } fn phrase_match(&mut self) -> bool { - // TODO early exit when we don't care about th phrase frequency - self.intersection_docset.docset(0).positions(&mut self.left); + // TODO early exit when we don't care about the phrase frequency + { + self.intersection_docset.docset_mut_specialized(0).positions(&mut self.left); + } let mut intersection_len = self.left.len(); for i in 1..self.num_docsets { - self.intersection_docset.docset(i).positions(&mut self.right); + { + self.intersection_docset.docset_mut_specialized(i).positions(&mut self.right); + } intersection_len = intersection_arr(&mut self.left[..intersection_len], &self.right[..]); if intersection_len == 0 { return false; From a3b44773bb93e50c9d94eba4a724f2156544a82d Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Sat, 10 Mar 2018 12:20:55 +0900 Subject: [PATCH 08/24] Bugfix and rustfmt --- src/core/index_meta.rs | 7 +++- src/core/inverted_index_reader.rs | 20 +++++++++-- src/core/segment_reader.rs | 47 +++++++++++++++++-------- src/datastruct/stacker/hashmap.rs | 5 ++- src/docset.rs | 4 --- src/fastfield/facet_reader.rs | 2 +- src/fastfield/mod.rs | 12 ++----- src/fastfield/multivalued/mod.rs | 7 ++-- src/fastfield/multivalued/reader.rs | 5 ++- src/fastfield/multivalued/writer.rs | 14 ++++---- src/fastfield/reader.rs | 9 ++--- src/fastfield/serializer.rs | 4 +-- src/fastfield/writer.rs | 2 +- src/lib.rs | 31 ++++++++++------ src/macros.rs | 2 +- src/query/boolean_query/mod.rs | 7 ++-- src/query/query_parser/query_grammar.rs | 3 +- src/query/query_parser/query_parser.rs | 8 +++-- src/query/scorer.rs | 6 ++-- src/query/union.rs | 4 ++- src/schema/schema.rs | 23 ++++++++---- src/termdict/fstdict/term_info_store.rs | 1 - src/termdict/fstdict/termdict.rs | 10 ++++++ src/termdict/mod.rs | 3 ++ 24 files changed, 150 insertions(+), 86 deletions(-) diff --git a/src/core/index_meta.rs b/src/core/index_meta.rs index 783f5aa8d..56db6fc62 100644 --- a/src/core/index_meta.rs +++ b/src/core/index_meta.rs @@ -33,7 +33,12 @@ impl IndexMeta { impl fmt::Debug for IndexMeta { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{}", serde_json::ser::to_string(self).expect("JSON serialization for IndexMeta should never fail.")) + write!( + f, + "{}", + serde_json::ser::to_string(self) + .expect("JSON serialization for IndexMeta should never fail.") + ) } } diff --git a/src/core/inverted_index_reader.rs b/src/core/inverted_index_reader.rs index cea7e4e4b..99c043b88 100644 --- a/src/core/inverted_index_reader.rs +++ b/src/core/inverted_index_reader.rs @@ -7,6 +7,7 @@ use schema::Term; use fastfield::DeleteBitSet; use compression::CompressedIntStream; use postings::FreqReadingOption; +use schema::FieldType; /// The inverted index reader is in charge of accessing /// the inverted index associated to a specific field. @@ -31,14 +32,14 @@ pub struct InvertedIndexReader { impl InvertedIndexReader { pub(crate) fn new( - termdict_source: ReadOnlySource, + termdict: TermDictionaryImpl, postings_source: ReadOnlySource, positions_source: ReadOnlySource, delete_bitset: DeleteBitSet, record_option: IndexRecordOption, ) -> InvertedIndexReader { InvertedIndexReader { - termdict: TermDictionaryImpl::from_source(termdict_source), + termdict, postings_source, positions_source, delete_bitset, @@ -46,6 +47,21 @@ impl InvertedIndexReader { } } + /// Creates an empty `InvertedIndexReader` object, which + /// contains no terms at all. + pub fn empty(field_type: FieldType) -> InvertedIndexReader { + let record_option = field_type + .get_index_record_option() + .unwrap_or(IndexRecordOption::Basic); + InvertedIndexReader::new( + TermDictionaryImpl::empty(field_type), + ReadOnlySource::empty(), + ReadOnlySource::empty(), + DeleteBitSet::empty(), + record_option, + ) + } + /// Returns the term info associated with the term. pub fn get_term_info(&self, term: &Term) -> Option { self.termdict.get(term.value_bytes()) diff --git a/src/core/segment_reader.rs b/src/core/segment_reader.rs index 85ddbfea0..059a2328b 100644 --- a/src/core/segment_reader.rs +++ b/src/core/segment_reader.rs @@ -8,7 +8,6 @@ use core::SegmentMeta; use fastfield::{self, FastFieldNotAvailableError}; use fastfield::DeleteBitSet; use store::StoreReader; -use directory::ReadOnlySource; use schema::Document; use DocId; use std::sync::Arc; @@ -97,7 +96,8 @@ impl SegmentReader { field: Field, ) -> fastfield::Result> { let field_entry = self.schema.get_field_entry(field); - if Item::fast_field_cardinality(field_entry.field_type()) == Some(Cardinality::SingleValue) { + if Item::fast_field_cardinality(field_entry.field_type()) == Some(Cardinality::SingleValue) + { self.fast_fields_composite .open_read(field) .ok_or_else(|| FastFieldNotAvailableError::new(field_entry)) @@ -109,9 +109,13 @@ impl SegmentReader { /// Accessor to the `MultiValueIntFastFieldReader` associated to a given `Field`. /// May panick if the field is not a multivalued fastfield of the type `Item`. - pub fn multi_fast_field_reader(&self, field: Field) -> fastfield::Result> { + pub fn multi_fast_field_reader( + &self, + field: Field, + ) -> fastfield::Result> { let field_entry = self.schema.get_field_entry(field); - if Item::fast_field_cardinality(field_entry.field_type()) == Some(Cardinality::MultiValues) { + if Item::fast_field_cardinality(field_entry.field_type()) == Some(Cardinality::MultiValues) + { let idx_reader = self.fast_fields_composite .open_read_with_idx(field, 0) .ok_or_else(|| FastFieldNotAvailableError::new(field_entry)) @@ -218,6 +222,8 @@ impl SegmentReader { } /// Returns a field reader associated to the field given in argument. + /// If the field was not present in the index during indexing time, + /// the InvertedIndexReader is empty. /// /// The field reader is in charge of iterating through the /// term dictionary associated to a specific field, @@ -230,27 +236,38 @@ impl SegmentReader { { return Arc::clone(inv_idx_reader); } + let field_entry = self.schema.get_field_entry(field); + let field_type = field_entry.field_type(); + let record_option_opt = field_type.get_index_record_option(); - let record_option = self.schema - .get_field_entry(field) - .field_type() - .get_index_record_option() - .expect("Field does not seem indexed."); + if record_option_opt.is_none() { + panic!("Field {:?} does not seem indexed.", field_entry.name()); + } - let termdict_source: ReadOnlySource = self.termdict_composite + let record_option = record_option_opt.unwrap(); + + let postings_source_opt = self.postings_composite.open_read(field); + + if postings_source_opt.is_none() { + // no documents in the segment contained this field. + // As a result, no data is associated to the inverted index. + // + // Returns an empty inverted index. + return Arc::new(InvertedIndexReader::empty(field_type.clone())); + } + + let postings_source = postings_source_opt.unwrap(); + + let termdict_source = self.termdict_composite .open_read(field) .expect("Failed to open field term dictionary in composite file. Is the field indexed"); - let postings_source = self.postings_composite - .open_read(field) - .expect("Index corrupted. Failed to open field postings in composite file."); - let positions_source = self.positions_composite .open_read(field) .expect("Index corrupted. Failed to open field positions in composite file."); let inv_idx_reader = Arc::new(InvertedIndexReader::new( - termdict_source, + TermDictionaryImpl::from_source(termdict_source), postings_source, positions_source, self.delete_bitset.clone(), diff --git a/src/datastruct/stacker/hashmap.rs b/src/datastruct/stacker/hashmap.rs index f1a61702c..a764ddc32 100644 --- a/src/datastruct/stacker/hashmap.rs +++ b/src/datastruct/stacker/hashmap.rs @@ -193,7 +193,10 @@ impl<'a> TermHashMap<'a> { } else if kv.hash == hash { let (stored_key, expull_addr): (&[u8], u32) = self.get_key_value(kv.key_value_addr); if stored_key == key_bytes { - return (bucket as UnorderedTermId, self.heap.get_mut_ref(expull_addr)); + return ( + bucket as UnorderedTermId, + self.heap.get_mut_ref(expull_addr), + ); } } } diff --git a/src/docset.rs b/src/docset.rs index 7fb93ac3a..356ae0b74 100644 --- a/src/docset.rs +++ b/src/docset.rs @@ -106,7 +106,6 @@ pub trait DocSet { } } - impl DocSet for Box { fn advance(&mut self) -> bool { let unboxed: &mut TDocSet = self.borrow_mut(); @@ -133,11 +132,8 @@ impl DocSet for Box { unboxed.count() } - fn append_to_bitset(&mut self, bitset: &mut BitSet) { let unboxed: &mut TDocSet = self.borrow_mut(); unboxed.append_to_bitset(bitset); } } - - diff --git a/src/fastfield/facet_reader.rs b/src/fastfield/facet_reader.rs index 049311094..d18a145b3 100644 --- a/src/fastfield/facet_reader.rs +++ b/src/fastfield/facet_reader.rs @@ -36,7 +36,7 @@ impl FacetReader { ) -> FacetReader { FacetReader { term_ords, - term_dict + term_dict, } } diff --git a/src/fastfield/mod.rs b/src/fastfield/mod.rs index 14d54f2a8..612c4e1b3 100644 --- a/src/fastfield/mod.rs +++ b/src/fastfield/mod.rs @@ -67,7 +67,6 @@ pub trait FastValue: Default + Clone + Copy { fn as_u64(&self) -> u64; } - impl FastValue for u64 { fn from_u64(val: u64) -> Self { val @@ -83,10 +82,8 @@ impl FastValue for u64 { fn fast_field_cardinality(field_type: &FieldType) -> Option { match *field_type { - FieldType::U64(ref integer_options) => - integer_options.get_fastfield_cardinality(), - FieldType::HierarchicalFacet => - Some(Cardinality::MultiValues), + FieldType::U64(ref integer_options) => integer_options.get_fastfield_cardinality(), + FieldType::HierarchicalFacet => Some(Cardinality::MultiValues), _ => None, } } @@ -101,11 +98,9 @@ impl FastValue for i64 { common::i64_to_u64(*self) } - fn fast_field_cardinality(field_type: &FieldType) -> Option { match *field_type { - FieldType::I64(ref integer_options) => - integer_options.get_fastfield_cardinality(), + FieldType::I64(ref integer_options) => integer_options.get_fastfield_cardinality(), _ => None, } } @@ -123,7 +118,6 @@ fn value_to_u64(value: &Value) -> u64 { } } - #[cfg(test)] mod tests { diff --git a/src/fastfield/multivalued/mod.rs b/src/fastfield/multivalued/mod.rs index 5c9f4dc9a..57a5db082 100644 --- a/src/fastfield/multivalued/mod.rs +++ b/src/fastfield/multivalued/mod.rs @@ -17,7 +17,7 @@ mod tests { let mut schema_builder = SchemaBuilder::default(); let field = schema_builder.add_u64_field( "multifield", - IntOptions::default().set_fast(Cardinality::MultiValues) + IntOptions::default().set_fast(Cardinality::MultiValues), ); let schema = schema_builder.build(); let index = Index::create_in_ram(schema); @@ -47,13 +47,12 @@ mod tests { } } - #[test] fn test_multivalued_i64() { let mut schema_builder = SchemaBuilder::default(); let field = schema_builder.add_i64_field( "multifield", - IntOptions::default().set_fast(Cardinality::MultiValues) + IntOptions::default().set_fast(Cardinality::MultiValues), ); let schema = schema_builder.build(); let index = Index::create_in_ram(schema); @@ -86,4 +85,4 @@ mod tests { assert_eq!(&vals, &[-5i64, -20i64, 1i64]); } } -} \ No newline at end of file +} diff --git a/src/fastfield/multivalued/reader.rs b/src/fastfield/multivalued/reader.rs index 4dbe49717..0fe2e12f3 100644 --- a/src/fastfield/multivalued/reader.rs +++ b/src/fastfield/multivalued/reader.rs @@ -1,7 +1,6 @@ use DocId; use fastfield::{FastFieldReader, FastValue}; - /// Reader for a multivalued `u64` fast field. /// /// The reader is implemented as two `u64` fast field. @@ -13,7 +12,7 @@ use fastfield::{FastFieldReader, FastValue}; #[derive(Clone)] pub struct MultiValueIntFastFieldReader { idx_reader: FastFieldReader, - vals_reader: FastFieldReader + vals_reader: FastFieldReader, } impl MultiValueIntFastFieldReader { @@ -23,7 +22,7 @@ impl MultiValueIntFastFieldReader { ) -> MultiValueIntFastFieldReader { MultiValueIntFastFieldReader { idx_reader, - vals_reader + vals_reader, } } diff --git a/src/fastfield/multivalued/writer.rs b/src/fastfield/multivalued/writer.rs index 15b3fa91e..593d2e964 100644 --- a/src/fastfield/multivalued/writer.rs +++ b/src/fastfield/multivalued/writer.rs @@ -7,12 +7,11 @@ use schema::{Document, Field}; use std::io; use itertools::Itertools; - pub struct MultiValueIntFastFieldWriter { field: Field, vals: Vec, doc_index: Vec, - is_facet: bool + is_facet: bool, } impl MultiValueIntFastFieldWriter { @@ -22,7 +21,7 @@ impl MultiValueIntFastFieldWriter { field, vals: Vec::new(), doc_index: Vec::new(), - is_facet + is_facet, } } @@ -51,7 +50,6 @@ impl MultiValueIntFastFieldWriter { } } } - } /// Serializes fast field values by pushing them to the `FastFieldSerializer`. @@ -85,8 +83,12 @@ impl MultiValueIntFastFieldWriter { let mut value_serializer: FastSingleFieldSerializer<_>; match mapping_opt { Some(mapping) => { - value_serializer = - serializer.new_u64_fast_field_with_idx(self.field, 0u64, mapping.len() as u64, 1)?; + value_serializer = serializer.new_u64_fast_field_with_idx( + self.field, + 0u64, + mapping.len() as u64, + 1, + )?; for val in &self.vals { let remapped_val = *mapping.get(val).expect("Missing term ordinal") as u64; value_serializer.add_val(remapped_val)?; diff --git a/src/fastfield/reader.rs b/src/fastfield/reader.rs index cfe0c2b77..389fdcfa9 100644 --- a/src/fastfield/reader.rs +++ b/src/fastfield/reader.rs @@ -24,11 +24,10 @@ pub struct FastFieldReader { bit_unpacker: BitUnpacker>, min_value_u64: u64, max_value_u64: u64, - _phantom: PhantomData + _phantom: PhantomData, } impl FastFieldReader { - /// Opens a fast field given a source. pub fn open(data: ReadOnlySource) -> Self { let min_value: u64; @@ -48,11 +47,10 @@ impl FastFieldReader { min_value_u64: min_value, max_value_u64: max_value, bit_unpacker, - _phantom: PhantomData + _phantom: PhantomData, } } - /// Return the value associated to the given document. /// /// This accessor should return as fast as possible. @@ -73,7 +71,7 @@ impl FastFieldReader { /// /// May panic if `start + output.len()` is greater than /// the segment's `maxdoc`. - pub fn get_range(&self, start: u32, output: &mut [Item]) { + pub fn get_range(&self, start: u32, output: &mut [Item]) { let output_u64: &mut [u64] = unsafe { mem::transmute(output) }; self.bit_unpacker.get_range(start, output_u64); for out in output_u64.iter_mut() { @@ -137,4 +135,3 @@ impl From> for FastFieldReader { FastFieldReader::open(field_source) } } - diff --git a/src/fastfield/serializer.rs b/src/fastfield/serializer.rs index 208b9e2ea..11dbfa7e9 100644 --- a/src/fastfield/serializer.rs +++ b/src/fastfield/serializer.rs @@ -36,9 +36,7 @@ impl FastFieldSerializer { pub fn from_write(write: WritePtr) -> io::Result { // just making room for the pointer to header. let composite_write = CompositeWrite::wrap(write); - Ok(FastFieldSerializer { - composite_write - }) + Ok(FastFieldSerializer { composite_write }) } /// Start serializing a new u64 fast field diff --git a/src/fastfield/writer.rs b/src/fastfield/writer.rs index 19cd5cef2..c708c3746 100644 --- a/src/fastfield/writer.rs +++ b/src/fastfield/writer.rs @@ -53,7 +53,7 @@ impl FastFieldsWriter { } FastFieldsWriter { single_value_writers, - multi_values_writers + multi_values_writers, } } diff --git a/src/lib.rs b/src/lib.rs index 82d1e0eae..077dbb871 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -645,6 +645,22 @@ mod tests { assert!(!postings.advance()); } + #[test] + fn test_indexedfield_not_in_documents() { + let mut schema_builder = SchemaBuilder::default(); + let text_field = schema_builder.add_text_field("text", TEXT); + let absent_field = schema_builder.add_text_field("text", TEXT); + let schema = schema_builder.build(); + let index = Index::create_in_ram(schema); + let mut index_writer = index.writer_with_num_threads(2, 40_000_000).unwrap(); + index_writer.add_document(doc!(text_field=>"a")); + assert!(index_writer.commit().is_ok()); + assert!(index.load_searchers().is_ok()); + let searcher = index.searcher(); + let segment_reader = searcher.segment_reader(0); + segment_reader.inverted_index(absent_field); //< should not panic + } + #[test] fn test_delete_postings2() { let mut schema_builder = SchemaBuilder::default(); @@ -859,31 +875,26 @@ mod tests { let searcher = index.searcher(); let segment_reader: &SegmentReader = searcher.segment_reader(0); { - let fast_field_reader_res = - segment_reader.fast_field_reader::(text_field); + let fast_field_reader_res = segment_reader.fast_field_reader::(text_field); assert!(fast_field_reader_res.is_err()); } { - let fast_field_reader_res = - segment_reader.fast_field_reader::(stored_int_field); + let fast_field_reader_res = segment_reader.fast_field_reader::(stored_int_field); assert!(fast_field_reader_res.is_err()); } { - let fast_field_reader_res = - segment_reader.fast_field_reader::(fast_field_signed); + let fast_field_reader_res = segment_reader.fast_field_reader::(fast_field_signed); assert!(fast_field_reader_res.is_err()); } { - let fast_field_reader_res = - segment_reader.fast_field_reader::(fast_field_signed); + let fast_field_reader_res = segment_reader.fast_field_reader::(fast_field_signed); assert!(fast_field_reader_res.is_ok()); let fast_field_reader = fast_field_reader_res.unwrap(); assert_eq!(fast_field_reader.get(0), 4i64) } { - let fast_field_reader_res = - segment_reader.fast_field_reader::(fast_field_signed); + let fast_field_reader_res = segment_reader.fast_field_reader::(fast_field_signed); assert!(fast_field_reader_res.is_ok()); let fast_field_reader = fast_field_reader_res.unwrap(); assert_eq!(fast_field_reader.get(0), 4i64) diff --git a/src/macros.rs b/src/macros.rs index cef2b1b5c..5e3d9b023 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -71,7 +71,7 @@ macro_rules! doc( #[cfg(test)] mod test { - use schema::{SchemaBuilder, TEXT, FAST}; + use schema::{SchemaBuilder, FAST, TEXT}; #[test] fn test_doc_basic() { diff --git a/src/query/boolean_query/mod.rs b/src/query/boolean_query/mod.rs index fd7393050..0435fb174 100644 --- a/src/query/boolean_query/mod.rs +++ b/src/query/boolean_query/mod.rs @@ -21,8 +21,6 @@ mod tests { use query::RequiredOptionalScorer; use query::score_combiner::SumWithCoordsCombiner; - - fn aux_test_helper() -> (Index, Field) { let mut schema_builder = SchemaBuilder::default(); let text_field = schema_builder.add_text_field("text", TEXT); @@ -104,7 +102,9 @@ mod tests { let query = query_parser.parse_query("+a b").unwrap(); let weight = query.weight(&*searcher, true).unwrap(); let scorer = weight.scorer(searcher.segment_reader(0u32)).unwrap(); - assert!(Downcast::, Box, SumWithCoordsCombiner>>::is_type(&*scorer)); + assert!(Downcast::< + RequiredOptionalScorer, Box, SumWithCoordsCombiner>, + >::is_type(&*scorer)); } { let query = query_parser.parse_query("+a b").unwrap(); @@ -116,7 +116,6 @@ mod tests { #[test] pub fn test_boolean_query() { - let (index, text_field) = aux_test_helper(); let make_term_query = |text: &str| { diff --git a/src/query/query_parser/query_grammar.rs b/src/query/query_parser/query_grammar.rs index 63691a6d9..aaee8bf46 100644 --- a/src/query/query_parser/query_grammar.rs +++ b/src/query/query_parser/query_grammar.rs @@ -41,7 +41,8 @@ fn leaf(input: I) -> ParseResult where I: Stream, { - (char('-'), parser(leaf)).map(|(_, expr)| UserInputAST::Not(box expr)) + (char('-'), parser(leaf)) + .map(|(_, expr)| UserInputAST::Not(box expr)) .or((char('+'), parser(leaf)).map(|(_, expr)| UserInputAST::Must(box expr))) .or((char('('), parser(parse_to_ast), char(')')).map(|(_, expr, _)| expr)) .or(parser(literal)) diff --git a/src/query/query_parser/query_parser.rs b/src/query/query_parser/query_parser.rs index 5e56aa886..d51225e5f 100644 --- a/src/query/query_parser/query_parser.rs +++ b/src/query/query_parser/query_parser.rs @@ -155,7 +155,7 @@ impl QueryParser { fn compute_logical_ast_for_leaf( &self, field: Field, - phrase: &str + phrase: &str, ) -> Result, QueryParserError> { let field_entry = self.schema.get_field_entry(field); let field_type = field_entry.field_type(); @@ -328,7 +328,7 @@ mod test { use tokenizer::TokenizerManager; use query::Query; use schema::Field; - use schema::{TextOptions, TextFieldIndexing, IndexRecordOption}; + use schema::{IndexRecordOption, TextFieldIndexing, TextOptions}; use super::QueryParser; use super::QueryParserError; use Index; @@ -538,7 +538,9 @@ mod test { let title = schema_builder.add_text_field("title", text_options); let schema = schema_builder.build(); let index = Index::create_in_ram(schema); - index.tokenizers().register("customtokenizer", SimpleTokenizer); + index + .tokenizers() + .register("customtokenizer", SimpleTokenizer); let query_parser = QueryParser::for_index(&index, vec![title]); assert!(query_parser.parse_query("title:\"happy tax\"").is_ok()); } diff --git a/src/query/scorer.rs b/src/query/scorer.rs index e1bba452f..7deadb0e5 100644 --- a/src/query/scorer.rs +++ b/src/query/scorer.rs @@ -51,8 +51,10 @@ impl DocSet for EmptyScorer { } fn doc(&self) -> DocId { - panic!("You may not call .doc() on a scorer \ - where the last call to advance() did not return true."); + panic!( + "You may not call .doc() on a scorer \ + where the last call to advance() did not return true." + ); } fn size_hint(&self) -> u32 { diff --git a/src/query/union.rs b/src/query/union.rs index 1f85b65e7..6b19eb34d 100644 --- a/src/query/union.rs +++ b/src/query/union.rs @@ -110,7 +110,9 @@ impl Union DocSet for Union - where TScorer: Scorer, TScoreCombiner: ScoreCombiner +where + TScorer: Scorer, + TScoreCombiner: ScoreCombiner, { fn advance(&mut self) -> bool { if self.advance_buffered() { diff --git a/src/schema/schema.rs b/src/schema/schema.rs index fb8188a06..99ddfbd47 100644 --- a/src/schema/schema.rs +++ b/src/schema/schema.rs @@ -123,7 +123,6 @@ impl Default for SchemaBuilder { } } - struct InnerSchema { fields: Vec, fields_map: HashMap, // transient @@ -243,7 +242,6 @@ impl Schema { } } - impl Serialize for Schema { fn serialize(&self, serializer: S) -> Result where @@ -292,7 +290,6 @@ impl<'de> Deserialize<'de> for Schema { } } - /// Error that may happen when deserializing /// a document from JSON. #[derive(Debug)] @@ -467,7 +464,10 @@ mod tests { "jambon": "bayonne" }"#, ); - assert_matches!(json_err, Err(DocParsingError::ValueError(_, ValueParsingError::TypeError(_)))); + assert_matches!( + json_err, + Err(DocParsingError::ValueError(_, ValueParsingError::TypeError(_))) + ); } { let json_err = schema.parse_document( @@ -478,7 +478,10 @@ mod tests { "popularity": 10 }"#, ); - assert_matches!(json_err, Err(DocParsingError::ValueError(_, ValueParsingError::OverflowError(_)))); + assert_matches!( + json_err, + Err(DocParsingError::ValueError(_, ValueParsingError::OverflowError(_))) + ); } { let json_err = schema.parse_document( @@ -489,7 +492,10 @@ mod tests { "popularity": 10 }"#, ); - assert!(!matches!(json_err, Err(DocParsingError::ValueError(_, ValueParsingError::OverflowError(_))))); + assert!(!matches!( + json_err, + Err(DocParsingError::ValueError(_, ValueParsingError::OverflowError(_))) + )); } { let json_err = schema.parse_document( @@ -500,7 +506,10 @@ mod tests { "popularity": 9223372036854775808 }"#, ); - assert_matches!(json_err, Err(DocParsingError::ValueError(_, ValueParsingError::OverflowError(_)))); + assert_matches!( + json_err, + Err(DocParsingError::ValueError(_, ValueParsingError::OverflowError(_))) + ); } { let json_err = schema.parse_document( diff --git a/src/termdict/fstdict/term_info_store.rs b/src/termdict/fstdict/term_info_store.rs index 1f7ca9779..289c71222 100644 --- a/src/termdict/fstdict/term_info_store.rs +++ b/src/termdict/fstdict/term_info_store.rs @@ -1,4 +1,3 @@ - use std::io; use std::cmp; use std::io::{Read, Write}; diff --git a/src/termdict/fstdict/termdict.rs b/src/termdict/fstdict/termdict.rs index 4583518c7..4b1d81c19 100644 --- a/src/termdict/fstdict/termdict.rs +++ b/src/termdict/fstdict/termdict.rs @@ -120,6 +120,16 @@ impl<'a> TermDictionary<'a> for TermDictionaryImpl { } } + fn empty(field_type: FieldType) -> Self { + let term_dictionary_data: Vec = + TermDictionaryBuilderImpl::new(Vec::::new(), field_type) + .expect("Creating a TermDictionaryBuilder in a Vec should never fail") + .finish() + .expect("Writing in a Vec should never fail"); + let source = ReadOnlySource::from(term_dictionary_data); + Self::from_source(source) + } + fn num_terms(&self) -> usize { self.term_info_store.num_terms() } diff --git a/src/termdict/mod.rs b/src/termdict/mod.rs index 64ff08732..abc4f6363 100644 --- a/src/termdict/mod.rs +++ b/src/termdict/mod.rs @@ -127,6 +127,9 @@ where .lt(stop_term.as_slice()) .into_stream() } + + /// Creates an empty term dictionary which contains no terms. + fn empty(field_type: FieldType) -> Self; } /// Builder for the new term dictionary. From 0cf274135b7f745fc1d22d0abc8b8b95168b9005 Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Sat, 10 Mar 2018 13:07:18 +0900 Subject: [PATCH 09/24] Clippy --- src/collector/top_collector.rs | 4 ++-- src/common/composite_file.rs | 14 ++++---------- src/common/timer.rs | 4 ++-- src/datastruct/stacker/hashmap.rs | 11 +++++------ src/indexer/prepared_commit.rs | 4 ++-- src/termdict/fstdict/streamer.rs | 4 ++-- src/termdict/mod.rs | 2 +- src/tokenizer/facet_tokenizer.rs | 2 +- 8 files changed, 19 insertions(+), 26 deletions(-) diff --git a/src/collector/top_collector.rs b/src/collector/top_collector.rs index 1a411b6d4..0fd6c61b5 100644 --- a/src/collector/top_collector.rs +++ b/src/collector/top_collector.rs @@ -60,7 +60,7 @@ impl TopCollector { panic!("Limit must be strictly greater than 0."); } TopCollector { - limit: limit, + limit, heap: BinaryHeap::with_capacity(limit), segment_id: 0, } @@ -119,7 +119,7 @@ impl Collector for TopCollector { } } else { let wrapped_doc = GlobalScoredDoc { - score: score, + score, doc_address: DocAddress(self.segment_id, doc), }; self.heap.push(wrapped_doc); diff --git a/src/common/composite_file.rs b/src/common/composite_file.rs index cceafce49..b30ed6403 100644 --- a/src/common/composite_file.rs +++ b/src/common/composite_file.rs @@ -16,10 +16,7 @@ pub struct FileAddr { impl FileAddr { fn new(field: Field, idx: usize) -> FileAddr { - FileAddr { - field: field, - idx: idx, - } + FileAddr { field, idx } } } @@ -34,8 +31,8 @@ impl BinarySerializable for FileAddr { let field = Field::deserialize(reader)?; let idx = VInt::deserialize(reader)?.0 as usize; Ok(FileAddr { - field: field, - idx: idx, + field, + idx, }) } } @@ -169,10 +166,7 @@ impl CompositeFile { /// to a given `Field` and stored in a `CompositeFile`. pub fn open_read_with_idx(&self, field: Field, idx: usize) -> Option { self.offsets_index - .get(&FileAddr { - field: field, - idx: idx, - }) + .get(&FileAddr { field, idx, }) .map(|&(from, to)| self.data.slice(from, to)) } } diff --git a/src/common/timer.rs b/src/common/timer.rs index 592c094a9..95638102d 100644 --- a/src/common/timer.rs +++ b/src/common/timer.rs @@ -14,7 +14,7 @@ impl<'a> OpenTimer<'a> { /// when the `OpenTimer` is dropped. pub fn open(&mut self, name: &'static str) -> OpenTimer { OpenTimer { - name: name, + name, timer_tree: self.timer_tree, start: PreciseTime::now(), depth: self.depth + 1, @@ -58,7 +58,7 @@ impl TimerTree { /// Open a new named subtask pub fn open(&mut self, name: &'static str) -> OpenTimer { OpenTimer { - name: name, + name, timer_tree: self, start: PreciseTime::now(), depth: 0, diff --git a/src/datastruct/stacker/hashmap.rs b/src/datastruct/stacker/hashmap.rs index a764ddc32..f0c17d5af 100644 --- a/src/datastruct/stacker/hashmap.rs +++ b/src/datastruct/stacker/hashmap.rs @@ -118,9 +118,9 @@ struct QuadraticProbing { impl QuadraticProbing { fn compute(hash: usize, mask: usize) -> QuadraticProbing { QuadraticProbing { - hash: hash, + hash, i: 0, - mask: mask, + mask, } } @@ -137,7 +137,7 @@ impl<'a> TermHashMap<'a> { let table: Vec = iter::repeat(KeyValue::default()).take(table_size).collect(); TermHashMap { table: table.into_boxed_slice(), - heap: heap, + heap, mask: table_size - 1, occupied: Vec::with_capacity(table_size / 2), } @@ -158,11 +158,10 @@ impl<'a> TermHashMap<'a> { (key_bytes, expull_addr) } - pub fn set_bucket(&mut self, hash: u32, key_bytes_ref: BytesRef, bucket: usize) { + pub fn set_bucket(&mut self, hash: u32, key_value_addr: BytesRef, bucket: usize) { self.occupied.push(bucket); self.table[bucket] = KeyValue { - key_value_addr: key_bytes_ref, - hash: hash, + key_value_addr, hash }; } diff --git a/src/indexer/prepared_commit.rs b/src/indexer/prepared_commit.rs index 02c79ed8b..b290b2c53 100644 --- a/src/indexer/prepared_commit.rs +++ b/src/indexer/prepared_commit.rs @@ -11,9 +11,9 @@ pub struct PreparedCommit<'a> { impl<'a> PreparedCommit<'a> { pub(crate) fn new(index_writer: &'a mut IndexWriter, opstamp: u64) -> PreparedCommit { PreparedCommit { - index_writer: index_writer, + index_writer, payload: None, - opstamp: opstamp, + opstamp } } diff --git a/src/termdict/fstdict/streamer.rs b/src/termdict/fstdict/streamer.rs index 4ebc3630c..1037610a6 100644 --- a/src/termdict/fstdict/streamer.rs +++ b/src/termdict/fstdict/streamer.rs @@ -13,8 +13,8 @@ pub struct TermStreamerBuilderImpl<'a> { impl<'a> TermStreamerBuilderImpl<'a> { pub(crate) fn new(fst_map: &'a TermDictionaryImpl, stream_builder: StreamBuilder<'a>) -> Self { TermStreamerBuilderImpl { - fst_map: fst_map, - stream_builder: stream_builder, + fst_map, + stream_builder, } } } diff --git a/src/termdict/mod.rs b/src/termdict/mod.rs index abc4f6363..05ca31240 100644 --- a/src/termdict/mod.rs +++ b/src/termdict/mod.rs @@ -127,7 +127,7 @@ where .lt(stop_term.as_slice()) .into_stream() } - + /// Creates an empty term dictionary which contains no terms. fn empty(field_type: FieldType) -> Self; } diff --git a/src/tokenizer/facet_tokenizer.rs b/src/tokenizer/facet_tokenizer.rs index 95b0d3711..fd408a4c4 100644 --- a/src/tokenizer/facet_tokenizer.rs +++ b/src/tokenizer/facet_tokenizer.rs @@ -31,7 +31,7 @@ impl<'a> Tokenizer<'a> for FacetTokenizer { fn token_stream(&self, text: &'a str) -> Self::TokenStreamImpl { FacetTokenStream { - text: text, + text, state: State::RootFacetNotEmitted, //< pos is the first char that has not been processed yet. token: Token::default(), } From 8683718159ea1a79c9aa29cfb91b8c992637a755 Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Sat, 10 Mar 2018 14:01:30 +0900 Subject: [PATCH 10/24] Version bump --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index f0e7b9526..632416a32 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tantivy" -version = "0.5.0" +version = "0.5.1" authors = ["Paul Masurel "] build = "build.rs" license = "MIT" From 97b7984200e5f658a7ab9afeb73ddbdf566abe45 Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Sat, 10 Mar 2018 14:08:11 +0900 Subject: [PATCH 11/24] Updated CHANGELOG --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c8bdc34a..fe8b4d947 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +Tantivy 0.5.1 +========================== +- bugfix #254 : tantivy failed if no documents in a segment contained a specific field. + + Tantivy 0.5 ========================== - Faceting From a2ee988304e82447dfe38761a04dbc84273066c5 Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Sat, 10 Mar 2018 15:32:30 +0900 Subject: [PATCH 12/24] Small change in pop_lowest. --- src/common/bitset.rs | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/src/common/bitset.rs b/src/common/bitset.rs index c0edea292..d8eef1336 100644 --- a/src/common/bitset.rs +++ b/src/common/bitset.rs @@ -1,4 +1,5 @@ use std::fmt; +use std::u64; #[derive(Clone, Copy, Eq, PartialEq)] pub(crate) struct TinySet(u64); @@ -84,23 +85,13 @@ impl TinySet { /// and removes it. #[inline(always)] pub fn pop_lowest(&mut self) -> Option { - if let Some(lowest) = self.lowest() { - self.0 ^= TinySet::singleton(lowest).0; - Some(lowest) - } else { - None - } - } - - /// Returns the lowest element in the `TinySet` - /// (or None if the set is empty). - #[inline(always)] - pub fn lowest(&mut self) -> Option { if self.is_empty() { None } else { - let least_significant_bit = self.0.trailing_zeros() as u32; - Some(least_significant_bit) + let least_significant_bit_pos = self.0.trailing_zeros() as u32; + let least_significant_bit = self.0 & self.0.wrapping_neg(); + self.0 ^= least_significant_bit; + Some(least_significant_bit_pos) } } @@ -366,7 +357,15 @@ mod tests { #[bench] fn bench_tinyset_pop(b: &mut test::Bencher) { - b.iter(|| test::black_box(TinySet::singleton(31u32)).pop_lowest()); + b.iter(|| { + let mut tinyset = TinySet::singleton(test::black_box(31u32)); + tinyset.pop_lowest(); + tinyset.pop_lowest(); + tinyset.pop_lowest(); + tinyset.pop_lowest(); + tinyset.pop_lowest(); + tinyset.pop_lowest(); + }); } #[bench] From d4ed86f13adc6e1c7ece40d5f76512d6c0212825 Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Mon, 19 Mar 2018 11:48:40 +0900 Subject: [PATCH 13/24] Issue/255 (#256) * Remove cpp compression. * Pointing to publish bitpacking * Edited README --- Cargo.toml | 5 +- README.md | 43 +- build.rs | 61 - cpp/simdcomp/.gitignore | 9 - cpp/simdcomp/.travis.yml | 11 - cpp/simdcomp/CHANGELOG | 9 - cpp/simdcomp/LICENSE | 27 - cpp/simdcomp/README.md | 137 - cpp/simdcomp/benchmarks/benchmark.c | 235 - cpp/simdcomp/benchmarks/bitpackingbenchmark.c | 205 - cpp/simdcomp/example.c | 195 - cpp/simdcomp/go/README.md | 13 - cpp/simdcomp/go/test.go | 71 - cpp/simdcomp/include/avxbitpacking.h | 40 - cpp/simdcomp/include/portability.h | 81 - cpp/simdcomp/include/simdbitpacking.h | 72 - cpp/simdcomp/include/simdcomp.h | 22 - cpp/simdcomp/include/simdcomputil.h | 54 - cpp/simdcomp/include/simdfor.h | 72 - .../include/simdintegratedbitpacking.h | 98 - cpp/simdcomp/makefile | 79 - cpp/simdcomp/makefile.vc | 104 - cpp/simdcomp/package.json | 16 - cpp/simdcomp/scripts/avxpacking.py | 182 - cpp/simdcomp/scripts/simdfor.py | 152 - cpp/simdcomp/simdcomp.def | 40 - cpp/simdcomp/src/avxbitpacking.c | 7795 ----- cpp/simdcomp/src/simdbitpacking.c | 14183 --------- cpp/simdcomp/src/simdcomputil.c | 234 - cpp/simdcomp/src/simdfor.c | 14501 --------- cpp/simdcomp/src/simdintegratedbitpacking.c | 24882 ---------------- cpp/simdcomp/src/simdpackedsearch.c | 15866 ---------- cpp/simdcomp/src/simdpackedselect.c | 15490 ---------- cpp/simdcomp/tests/unit.c | 900 - cpp/simdcomp/tests/unit_chars.c | 102 - cpp/simdcomp_wrapper.c | 42 - cpp/streamvbyte/.gitignore | 32 - cpp/streamvbyte/.travis.yml | 7 - cpp/streamvbyte/LICENSE | 202 - cpp/streamvbyte/README.md | 60 - cpp/streamvbyte/example.c | 24 - cpp/streamvbyte/include/streamvbyte.h | 19 - cpp/streamvbyte/include/streamvbytedelta.h | 24 - cpp/streamvbyte/makefile | 58 - cpp/streamvbyte/src/streamvbyte.c | 495 - cpp/streamvbyte/src/streamvbytedelta.c | 575 - cpp/streamvbyte/tests/unit.c | 73 - src/compression/mod.rs | 137 +- .../pack/compression_pack_nosimd.rs | 148 - src/compression/pack/compression_pack_simd.rs | 116 - src/compression/stream.rs | 4 +- .../compression_vint_nosimd.rs => vint.rs} | 0 src/compression/vint/compression_vint_simd.rs | 72 - src/lib.rs | 4 +- 54 files changed, 157 insertions(+), 97921 deletions(-) delete mode 100644 build.rs delete mode 100644 cpp/simdcomp/.gitignore delete mode 100644 cpp/simdcomp/.travis.yml delete mode 100644 cpp/simdcomp/CHANGELOG delete mode 100644 cpp/simdcomp/LICENSE delete mode 100644 cpp/simdcomp/README.md delete mode 100644 cpp/simdcomp/benchmarks/benchmark.c delete mode 100644 cpp/simdcomp/benchmarks/bitpackingbenchmark.c delete mode 100644 cpp/simdcomp/example.c delete mode 100644 cpp/simdcomp/go/README.md delete mode 100644 cpp/simdcomp/go/test.go delete mode 100644 cpp/simdcomp/include/avxbitpacking.h delete mode 100644 cpp/simdcomp/include/portability.h delete mode 100644 cpp/simdcomp/include/simdbitpacking.h delete mode 100644 cpp/simdcomp/include/simdcomp.h delete mode 100644 cpp/simdcomp/include/simdcomputil.h delete mode 100644 cpp/simdcomp/include/simdfor.h delete mode 100644 cpp/simdcomp/include/simdintegratedbitpacking.h delete mode 100644 cpp/simdcomp/makefile delete mode 100644 cpp/simdcomp/makefile.vc delete mode 100644 cpp/simdcomp/package.json delete mode 100755 cpp/simdcomp/scripts/avxpacking.py delete mode 100755 cpp/simdcomp/scripts/simdfor.py delete mode 100644 cpp/simdcomp/simdcomp.def delete mode 100644 cpp/simdcomp/src/avxbitpacking.c delete mode 100644 cpp/simdcomp/src/simdbitpacking.c delete mode 100644 cpp/simdcomp/src/simdcomputil.c delete mode 100644 cpp/simdcomp/src/simdfor.c delete mode 100644 cpp/simdcomp/src/simdintegratedbitpacking.c delete mode 100644 cpp/simdcomp/src/simdpackedsearch.c delete mode 100644 cpp/simdcomp/src/simdpackedselect.c delete mode 100644 cpp/simdcomp/tests/unit.c delete mode 100644 cpp/simdcomp/tests/unit_chars.c delete mode 100644 cpp/simdcomp_wrapper.c delete mode 100644 cpp/streamvbyte/.gitignore delete mode 100644 cpp/streamvbyte/.travis.yml delete mode 100644 cpp/streamvbyte/LICENSE delete mode 100644 cpp/streamvbyte/README.md delete mode 100644 cpp/streamvbyte/example.c delete mode 100644 cpp/streamvbyte/include/streamvbyte.h delete mode 100644 cpp/streamvbyte/include/streamvbytedelta.h delete mode 100644 cpp/streamvbyte/makefile delete mode 100644 cpp/streamvbyte/src/streamvbyte.c delete mode 100644 cpp/streamvbyte/src/streamvbytedelta.c delete mode 100644 cpp/streamvbyte/tests/unit.c delete mode 100644 src/compression/pack/compression_pack_nosimd.rs delete mode 100644 src/compression/pack/compression_pack_simd.rs rename src/compression/{vint/compression_vint_nosimd.rs => vint.rs} (100%) delete mode 100644 src/compression/vint/compression_vint_simd.rs diff --git a/Cargo.toml b/Cargo.toml index 632416a32..9776a4de1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,6 @@ name = "tantivy" version = "0.5.1" authors = ["Paul Masurel "] -build = "build.rs" license = "MIT" categories = ["database-implementations", "data-structures"] description = """Tantivy is a search engine library.""" @@ -26,7 +25,6 @@ tempdir = "0.3" serde = "1.0" serde_derive = "1.0" serde_json = "1.0" -libc = { version = "0.2.20", optional=true } num_cpus = "1.2" itertools = "0.5.9" lz4 = "1.20" @@ -43,6 +41,7 @@ stable_deref_trait = "1.0.0" rust-stemmers = "0.1.0" downcast = { version="0.9", features = ["nightly"]} matches = "0.1" +bitpacking = "0.1" [target.'cfg(windows)'.dependencies] winapi = "0.2" @@ -63,7 +62,7 @@ debug-assertions = false [features] default = ["simdcompression"] -simdcompression = ["libc", "cc"] +simdcompression = [] streamdict = [] diff --git a/README.md b/README.md index d17e5b03e..591798b93 100644 --- a/README.md +++ b/README.md @@ -5,25 +5,26 @@ [![Join the chat at https://gitter.im/tantivy-search/tantivy](https://badges.gitter.im/tantivy-search/tantivy.svg)](https://gitter.im/tantivy-search/tantivy?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![Build status](https://ci.appveyor.com/api/projects/status/r7nb13kj23u8m9pj?svg=true)](https://ci.appveyor.com/project/fulmicoton/tantivy) -![beacon for google analytics](https://ga-beacon.appspot.com/UA-88834340-1/tantivy/README) **Tantivy** is a **full text search engine library** written in rust. It is strongly inspired by Lucene's design. - # Features -- configurable indexing (optional term frequency and position indexing) +- Tiny startup time (<10ms), perfect for command line tools - tf-idf scoring - Basic query language - Phrase queries - Incremental indexing - Multithreaded indexing (indexing English Wikipedia takes < 3 minutes on my desktop) -- mmap based +- Mmap directory - optional SIMD integer compression -- u64 and i64 fast fields (equivalent of doc values in Lucene) +- Single valued and multivalued u64 and i64 fast fields (equivalent of doc values in Lucene) - LZ4 compressed document store +- Range queries +- Faceting +- configurable indexing (optional term frequency and position indexing - Cheesy logo with a horse Tantivy supports Linux, MacOS and Windows. @@ -40,14 +41,38 @@ It will walk you through getting a wikipedia search engine up and running in a f # Compiling -Tantivy requires Rust Nightly because it uses requires the features [`box_syntax`](https://doc.rust-lang.org/stable/book/box-syntax-and-patterns.html), [`optin_builtin_traits`](https://github.com/rust-lang/rfcs/blob/master/text/0019-opt-in-builtin-traits.md), and [`conservative_impl_trait`](https://github.com/rust-lang/rfcs/blob/master/text/1522-conservative-impl-trait.md). -The project can then be built using `cargo`. +## Development + +Tantivy requires Rust Nightly because it uses requires the features [`box_syntax`](https://doc.rust-lang.org/stable/unstable-book/language-features/box-syntax.html), [`optin_builtin_traits`](https://github.com/rust-lang/rfcs/blob/master/text/0019-opt-in-builtin-traits.md), [`conservative_impl_trait`](https://github.com/rust-lang/rfcs/blob/master/text/1522-conservative-impl-trait.md), +and [simd](https://github.com/rust-lang/rust/issues/27731). + + +To check out and run test, you can simply run : git clone git@github.com:tantivy-search/tantivy.git cd tantivy - cargo build + cargo +nightly build +## Note on release build and performance + +If your project depends on `tantivy`, for better performance, make sure to enable +`sse3` instructions using a RUSTFLAGS. (This instruction set is likely to +be available on most `x86_64` CPUs you will encounter). + +For instance, + + RUSTFLAGS='-C target-feature=+sse3' + +Or, if you are targetting a specific cpu + + RUSTFLAGS='-C target-cpu=native' build --release + +Regardless of the flags you pass, by default `tantivy` will contain `SSE3` instructions. +If you want to disable those, you can run the following command : + + cargo build --no-default-features + Alternatively, if you are trying to compile `tantivy` without simd compression, you can disable this functionality. In this case, this submodule is not required and you can compile tantivy by using the `--no-default-features` flag. @@ -57,4 +82,4 @@ and you can compile tantivy by using the `--no-default-features` flag. # Contribute -Send me an email (paul.masurel at gmail.com) if you want to contribute to tantivy. +Send me an email (paul.masurel at gmail.com) if you want to contribute to tantivy. \ No newline at end of file diff --git a/build.rs b/build.rs deleted file mode 100644 index 7fda7e816..000000000 --- a/build.rs +++ /dev/null @@ -1,61 +0,0 @@ -#[cfg(feature = "simdcompression")] -mod build { - extern crate cc; - - pub fn build() { - let mut config = cc::Build::new(); - config - .include("./cpp/simdcomp/include") - .file("cpp/simdcomp/src/avxbitpacking.c") - .file("cpp/simdcomp/src/simdintegratedbitpacking.c") - .file("cpp/simdcomp/src/simdbitpacking.c") - .file("cpp/simdcomp/src/simdpackedsearch.c") - .file("cpp/simdcomp/src/simdcomputil.c") - .file("cpp/simdcomp/src/simdpackedselect.c") - .file("cpp/simdcomp/src/simdfor.c") - .file("cpp/simdcomp_wrapper.c"); - - if !cfg!(debug_assertions) { - config.opt_level(3); - - if cfg!(target_env = "msvc") { - config - .define("NDEBUG", None) - .flag("/Gm-") - .flag("/GS-") - .flag("/Gy") - .flag("/Oi") - .flag("/GL"); - } - } - - if !cfg!(target_env = "msvc") { - config - .include("./cpp/streamvbyte/include") - .file("cpp/streamvbyte/src/streamvbyte.c") - .file("cpp/streamvbyte/src/streamvbytedelta.c") - .flag("-msse4.1") - .flag("-march=native") - .flag("-std=c99"); - } - - config.compile("libsimdcomp.a"); - - // Workaround for linking static libraries built with /GL - // https://github.com/rust-lang/rust/issues/26003 - if !cfg!(debug_assertions) && cfg!(target_env = "msvc") { - println!("cargo:rustc-link-lib=dylib=simdcomp"); - } - - println!("cargo:rerun-if-changed=cpp"); - } -} - -#[cfg(not(feature = "simdcompression"))] -mod build { - pub fn build() {} -} - -fn main() { - build::build(); -} diff --git a/cpp/simdcomp/.gitignore b/cpp/simdcomp/.gitignore deleted file mode 100644 index b8334280c..000000000 --- a/cpp/simdcomp/.gitignore +++ /dev/null @@ -1,9 +0,0 @@ -Makefile.in -lib* -unit* -*.o -src/*.lo -src/*.o -src/.deps -src/.dirstamp -src/.libs diff --git a/cpp/simdcomp/.travis.yml b/cpp/simdcomp/.travis.yml deleted file mode 100644 index 28f13df7a..000000000 --- a/cpp/simdcomp/.travis.yml +++ /dev/null @@ -1,11 +0,0 @@ -language: c -sudo: false -compiler: - - gcc - - clang - -branches: - only: - - master - -script: make && ./unit diff --git a/cpp/simdcomp/CHANGELOG b/cpp/simdcomp/CHANGELOG deleted file mode 100644 index bf56786f4..000000000 --- a/cpp/simdcomp/CHANGELOG +++ /dev/null @@ -1,9 +0,0 @@ -Upcoming - - added missing include - - improved portability (MSVC) - - implemented C89 compatibility -Version 0.0.3 (19 May 2014) - - improved documentation -Version 0.0.2 (6 February 2014) - - added go demo -Version 0.0.1 (5 February 2014) diff --git a/cpp/simdcomp/LICENSE b/cpp/simdcomp/LICENSE deleted file mode 100644 index f3c5904df..000000000 --- a/cpp/simdcomp/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2014--, The authors -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, this - list of conditions and the following disclaimer in the documentation and/or - other materials provided with the distribution. - -* Neither the name of the {organization} nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/cpp/simdcomp/README.md b/cpp/simdcomp/README.md deleted file mode 100644 index c8c45d1e2..000000000 --- a/cpp/simdcomp/README.md +++ /dev/null @@ -1,137 +0,0 @@ -The SIMDComp library -==================== -[![Build Status](https://travis-ci.org/lemire/simdcomp.png)](https://travis-ci.org/lemire/simdcomp) - -A simple C library for compressing lists of integers using binary packing and SIMD instructions. -The assumption is either that you have a list of 32-bit integers where most of them are small, or a list of 32-bit integers where differences between successive integers are small. No software is able to reliably compress an array of 32-bit random numbers. - -This library can decode at least 4 billions of compressed integers per second on most -desktop or laptop processors. That is, it can decompress data at a rate of 15 GB/s. -This is significantly faster than generic codecs like gzip, LZO, Snappy or LZ4. - -On a Skylake Intel processor, it can decode integers at a rate 0.3 cycles per integer, -which can easily translate into more than 8 decoded billions integers per second. - -Contributors: Daniel Lemire, Nathan Kurz, Christoph Rupp, Anatol Belski, Nick White and others - -What is it for? -------------- - -This is a low-level library for fast integer compression. By design it does not define a compressed -format. It is up to the (sophisticated) user to create a compressed format. - -Requirements -------------- - -- Your processor should support SSE4.1 (It is supported by most Intel and AMD processors released since 2008.) -- It is possible to build the core part of the code if your processor support SSE2 (Pentium4 or better) -- C99 compliant compiler (GCC is assumed) -- A Linux-like distribution is assumed by the makefile - -For a plain C version that does not use SIMD instructions, see https://github.com/lemire/LittleIntPacker - -Usage -------- - -Compression works over blocks of 128 integers. - -For a complete working example, see example.c (you can build it and -run it with "make example; ./example"). - - - -1) Lists of integers in random order. - -```C -const uint32_t b = maxbits(datain);// computes bit width -simdpackwithoutmask(datain, buffer, b);//compressed to buffer, compressing 128 32-bit integers down to b*32 bytes -simdunpack(buffer, backbuffer, b);//uncompressed to backbuffer -``` - -While 128 32-bit integers are read, only b 128-bit words are written. Thus, the compression ratio is 32/b. - -2) Sorted lists of integers. - -We used differential coding: we store the difference between successive integers. For this purpose, we need an initial value (called offset). - -```C -uint32_t offset = 0; -uint32_t b1 = simdmaxbitsd1(offset,datain); // bit width -simdpackwithoutmaskd1(offset, datain, buffer, b1);//compressing 128 32-bit integers down to b1*32 bytes -simdunpackd1(offset, buffer, backbuffer, b1);//uncompressed -``` - -General example for arrays of arbitrary length: -```C -int compress_decompress_demo() { - size_t k, N = 9999; - __m128i * endofbuf; - uint32_t * datain = malloc(N * sizeof(uint32_t)); - uint8_t * buffer; - uint32_t * backbuffer = malloc(N * sizeof(uint32_t)); - uint32_t b; - - for (k = 0; k < N; ++k){ /* start with k=0, not k=1! */ - datain[k] = k; - } - - b = maxbits_length(datain, N); - buffer = malloc(simdpack_compressedbytes(N,b)); // allocate just enough memory - endofbuf = simdpack_length(datain, N, (__m128i *)buffer, b); - /* compressed data is stored between buffer and endofbuf using (endofbuf-buffer)*sizeof(__m128i) bytes */ - /* would be safe to do : buffer = realloc(buffer,(endofbuf-(__m128i *)buffer)*sizeof(__m128i)); */ - simdunpack_length((const __m128i *)buffer, N, backbuffer, b); - - for (k = 0; k < N; ++k){ - if(datain[k] != backbuffer[k]) { - printf("bug\n"); - return -1; - } - } - return 0; -} -``` - - -3) Frame-of-Reference - -We also have frame-of-reference (FOR) functions (see simdfor.h header). They work like the bit packing -routines, but do not use differential coding so they allow faster search in some cases, at the expense -of compression. - -Setup ---------- - - -make -make test - -and if you are daring: - -make install - -Go --------- - -If you are a go user, there is a "go" folder where you will find a simple demo. - -Other libraries ----------------- - -* Fast decoder for VByte-compressed integers https://github.com/lemire/MaskedVByte -* Fast integer compression in C using StreamVByte https://github.com/lemire/streamvbyte -* FastPFOR is a C++ research library well suited to compress unsorted arrays: https://github.com/lemire/FastPFor -* SIMDCompressionAndIntersection is a C++ research library well suited for sorted arrays (differential coding) -and computing intersections: https://github.com/lemire/SIMDCompressionAndIntersection -* TurboPFor is a C library that offers lots of interesting optimizations. Well worth checking! (GPL license) https://github.com/powturbo/TurboPFor -* Oroch is a C++ library that offers a usable API (MIT license) https://github.com/ademakov/Oroch - - -References ------------- - -* Daniel Lemire, Leonid Boytsov, Nathan Kurz, SIMD Compression and the Intersection of Sorted Integers, Software Practice & Experience 46 (6) 2016. http://arxiv.org/abs/1401.6399 -* Daniel Lemire and Leonid Boytsov, Decoding billions of integers per second through vectorization, Software Practice & Experience 45 (1), 2015. http://arxiv.org/abs/1209.2137 http://onlinelibrary.wiley.com/doi/10.1002/spe.2203/abstract -* Jeff Plaisance, Nathan Kurz, Daniel Lemire, Vectorized VByte Decoding, International Symposium on Web Algorithms 2015, 2015. http://arxiv.org/abs/1503.07387 -* Wayne Xin Zhao, Xudong Zhang, Daniel Lemire, Dongdong Shan, Jian-Yun Nie, Hongfei Yan, Ji-Rong Wen, A General SIMD-based Approach to Accelerating Compression Algorithms, ACM Transactions on Information Systems 33 (3), 2015. http://arxiv.org/abs/1502.01916 -* T. D. Wu, Bitpacking techniques for indexing genomes: I. Hash tables, Algorithms for Molecular Biology 11 (5), 2016. http://almob.biomedcentral.com/articles/10.1186/s13015-016-0069-5 diff --git a/cpp/simdcomp/benchmarks/benchmark.c b/cpp/simdcomp/benchmarks/benchmark.c deleted file mode 100644 index 783ef9485..000000000 --- a/cpp/simdcomp/benchmarks/benchmark.c +++ /dev/null @@ -1,235 +0,0 @@ -/** - * This code is released under a BSD License. - */ -#include -#include -#include -#include - -#include "simdcomp.h" - -#ifdef _MSC_VER -# include - -__int64 freq; - -typedef __int64 time_snap_t; - -static time_snap_t time_snap(void) -{ - __int64 now; - - QueryPerformanceCounter((LARGE_INTEGER *)&now); - - return (__int64)((now*1000000)/freq); -} -# define TIME_SNAP_FMT "%I64d" -#else -# define time_snap clock -# define TIME_SNAP_FMT "%lu" -typedef clock_t time_snap_t; -#endif - - -void benchmarkSelect() { - uint32_t buffer[128]; - uint32_t backbuffer[128]; - uint32_t initial = 33; - uint32_t b; - time_snap_t S1, S2, S3; - int i; - printf("benchmarking select \n"); - - /* this test creates delta encoded buffers with different bits, then - * performs lower bound searches for each key */ - for (b = 0; b <= 32; b++) { - uint32_t prev = initial; - uint32_t out[128]; - /* initialize the buffer */ - for (i = 0; i < 128; i++) { - buffer[i] = ((uint32_t)(1655765 * i )) ; - if(b < 32) buffer[i] %= (1< *ib) - return 1; - return 0; -} - -/* adapted from wikipedia */ -int binary_search(uint32_t * A, uint32_t key, int imin, int imax) -{ - int imid; - imax --; - while(imin + 1 < imax) { - imid = imin + ((imax - imin) / 2); - - if (A[imid] > key) { - imax = imid; - } else if (A[imid] < key) { - imin = imid; - } else { - return imid; - } - } - return imax; -} - - -/* adapted from wikipedia */ -int lower_bound(uint32_t * A, uint32_t key, int imin, int imax) -{ - int imid; - imax --; - while(imin + 1 < imax) { - imid = imin + ((imax - imin) / 2); - - if (A[imid] >= key) { - imax = imid; - } else if (A[imid] < key) { - imin = imid; - } - } - if(A[imin] >= key) return imin; - return imax; -} - -void benchmarkSearch() { - uint32_t buffer[128]; - uint32_t backbuffer[128]; - uint32_t out[128]; - uint32_t result, initial = 0; - uint32_t b, i; - time_snap_t S1, S2, S3, S4; - - printf("benchmarking search \n"); - - /* this test creates delta encoded buffers with different bits, then - * performs lower bound searches for each key */ - for (b = 0; b <= 32; b++) { - uint32_t prev = initial; - /* initialize the buffer */ - for (i = 0; i < 128; i++) { - buffer[i] = ((uint32_t)rand()) ; - if(b < 32) buffer[i] %= (1< 0) { - if(buffer[pos-1] >= pseudorandomkey) - printf("bug B.\n"); - } - } - S2 = time_snap(); - for (i = 0; i < 128 * 10; i++) { - int pos; - uint32_t pseudorandomkey = buffer[i%128]; - simdunpackd1(initial, (__m128i *)out, backbuffer, b); - pos = lower_bound(backbuffer, pseudorandomkey, 0, 128); - result = backbuffer[pos]; - - if((result < pseudorandomkey) || (buffer[pos] != result)) { - printf("bug C.\n"); - } else if (pos > 0) { - if(buffer[pos-1] >= pseudorandomkey) - printf("bug D.\n"); - } - } - S3 = time_snap(); - for (i = 0; i < 128 * 10; i++) { - - int pos; - uint32_t pseudorandomkey = buffer[i%128]; - pos = simdsearchwithlengthd1(initial, (__m128i *)out, b, 128, - pseudorandomkey, &result); - if((result < pseudorandomkey) || (buffer[pos] != result)) { - printf("bug A.\n"); - } else if (pos > 0) { - if(buffer[pos-1] >= pseudorandomkey) - printf("bug B.\n"); - } - } - S4 = time_snap(); - - printf("bit width = %d, fast search function time = " TIME_SNAP_FMT ", naive time = " TIME_SNAP_FMT " , fast with length time = " TIME_SNAP_FMT " \n", b, (S2-S1), (S3-S2), (S4-S3) ); - } -} - - -int main() { -#ifdef _MSC_VER - QueryPerformanceFrequency((LARGE_INTEGER *)&freq); -#endif - benchmarkSearch(); - benchmarkSelect(); - return 0; -} diff --git a/cpp/simdcomp/benchmarks/bitpackingbenchmark.c b/cpp/simdcomp/benchmarks/bitpackingbenchmark.c deleted file mode 100644 index 4426d9897..000000000 --- a/cpp/simdcomp/benchmarks/bitpackingbenchmark.c +++ /dev/null @@ -1,205 +0,0 @@ -#include - -#include "simdcomp.h" - - -#define RDTSC_START(cycles) \ - do { \ - register unsigned cyc_high, cyc_low; \ - __asm volatile( \ - "cpuid\n\t" \ - "rdtsc\n\t" \ - "mov %%edx, %0\n\t" \ - "mov %%eax, %1\n\t" \ - : "=r"(cyc_high), "=r"(cyc_low)::"%rax", "%rbx", "%rcx", "%rdx"); \ - (cycles) = ((uint64_t)cyc_high << 32) | cyc_low; \ - } while (0) - -#define RDTSC_FINAL(cycles) \ - do { \ - register unsigned cyc_high, cyc_low; \ - __asm volatile( \ - "rdtscp\n\t" \ - "mov %%edx, %0\n\t" \ - "mov %%eax, %1\n\t" \ - "cpuid\n\t" \ - : "=r"(cyc_high), "=r"(cyc_low)::"%rax", "%rbx", "%rcx", "%rdx"); \ - (cycles) = ((uint64_t)cyc_high << 32) | cyc_low; \ - } while (0) - - - - -uint32_t * get_random_array_from_bit_width(uint32_t length, uint32_t bit) { - uint32_t * answer = malloc(sizeof(uint32_t) * length); - uint32_t mask = (uint32_t) ((UINT64_C(1) << bit) - 1); - uint32_t i; - for(i = 0; i < length; ++i) { - answer[i] = rand() & mask; - } - return answer; -} - -uint32_t * get_random_array_from_bit_width_d1(uint32_t length, uint32_t bit) { - uint32_t * answer = malloc(sizeof(uint32_t) * length); - uint32_t mask = (uint32_t) ((UINT64_C(1) << bit) - 1); - uint32_t i; - answer[0] = rand() & mask; - for(i = 1; i < length; ++i) { - answer[i] = answer[i-1] + (rand() & mask); - } - return answer; -} - - -void demo128() { - const uint32_t length = 128; - uint32_t bit; - printf("# --- %s\n", __func__); - printf("# compressing %d integers\n",length); - printf("# format: bit width, pack in cycles per int, unpack in cycles per int\n"); - for(bit = 1; bit <= 32; ++bit) { - uint32_t i; - - uint32_t * data = get_random_array_from_bit_width(length, bit); - __m128i * buffer = malloc(length * sizeof(uint32_t)); - uint32_t * backdata = malloc(length * sizeof(uint32_t)); - uint32_t repeat = 500; - uint64_t min_diff; - printf("%d\t",bit); - min_diff = (uint64_t)-1; - for (i = 0; i < repeat; i++) { - uint64_t cycles_start, cycles_final, cycles_diff; - __asm volatile("" ::: /* pretend to clobber */ "memory"); - RDTSC_START(cycles_start); - simdpackwithoutmask(data,buffer, bit); - RDTSC_FINAL(cycles_final); - cycles_diff = (cycles_final - cycles_start); - if (cycles_diff < min_diff) min_diff = cycles_diff; - } - printf("%.2f\t",min_diff*1.0/length); - min_diff = (uint64_t)-1; - for (i = 0; i < repeat; i++) { - uint64_t cycles_start, cycles_final, cycles_diff; - __asm volatile("" ::: /* pretend to clobber */ "memory"); - RDTSC_START(cycles_start); - simdunpack(buffer, backdata,bit); - RDTSC_FINAL(cycles_final); - cycles_diff = (cycles_final - cycles_start); - if (cycles_diff < min_diff) min_diff = cycles_diff; - } - printf("%.2f\t",min_diff*1.0/length); - - free(data); - free(buffer); - free(backdata); - printf("\n"); - } - printf("\n\n"); /* two blank lines are required by gnuplot */ -} - -void demo128_d1() { - const uint32_t length = 128; - uint32_t bit; - printf("# --- %s\n", __func__); - printf("# compressing %d integers\n",length); - printf("# format: bit width, pack in cycles per int, unpack in cycles per int\n"); - for(bit = 1; bit <= 32; ++bit) { - uint32_t i; - - uint32_t * data = get_random_array_from_bit_width_d1(length, bit); - __m128i * buffer = malloc(length * sizeof(uint32_t)); - uint32_t * backdata = malloc(length * sizeof(uint32_t)); - uint32_t repeat = 500; - uint64_t min_diff; - printf("%d\t",bit); - min_diff = (uint64_t)-1; - for (i = 0; i < repeat; i++) { - uint64_t cycles_start, cycles_final, cycles_diff; - __asm volatile("" ::: /* pretend to clobber */ "memory"); - RDTSC_START(cycles_start); - simdpackwithoutmaskd1(0,data,buffer, bit); - RDTSC_FINAL(cycles_final); - cycles_diff = (cycles_final - cycles_start); - if (cycles_diff < min_diff) min_diff = cycles_diff; - } - printf("%.2f\t",min_diff*1.0/length); - min_diff = (uint64_t)-1; - for (i = 0; i < repeat; i++) { - uint64_t cycles_start, cycles_final, cycles_diff; - __asm volatile("" ::: /* pretend to clobber */ "memory"); - RDTSC_START(cycles_start); - simdunpackd1(0,buffer, backdata,bit); - RDTSC_FINAL(cycles_final); - cycles_diff = (cycles_final - cycles_start); - if (cycles_diff < min_diff) min_diff = cycles_diff; - } - printf("%.2f\t",min_diff*1.0/length); - - free(data); - free(buffer); - free(backdata); - printf("\n"); - } - printf("\n\n"); /* two blank lines are required by gnuplot */ -} - -#ifdef __AVX2__ -void demo256() { - const uint32_t length = 256; - uint32_t bit; - printf("# --- %s\n", __func__); - printf("# compressing %d integers\n",length); - printf("# format: bit width, pack in cycles per int, unpack in cycles per int\n"); - for(bit = 1; bit <= 32; ++bit) { - uint32_t i; - - uint32_t * data = get_random_array_from_bit_width(length, bit); - __m256i * buffer = malloc(length * sizeof(uint32_t)); - uint32_t * backdata = malloc(length * sizeof(uint32_t)); - uint32_t repeat = 500; - uint64_t min_diff; - printf("%d\t",bit); - min_diff = (uint64_t)-1; - for (i = 0; i < repeat; i++) { - uint64_t cycles_start, cycles_final, cycles_diff; - __asm volatile("" ::: /* pretend to clobber */ "memory"); - RDTSC_START(cycles_start); - avxpackwithoutmask(data,buffer, bit); - RDTSC_FINAL(cycles_final); - cycles_diff = (cycles_final - cycles_start); - if (cycles_diff < min_diff) min_diff = cycles_diff; - } - printf("%.2f\t",min_diff*1.0/length); - min_diff = (uint64_t)-1; - for (i = 0; i < repeat; i++) { - uint64_t cycles_start, cycles_final, cycles_diff; - __asm volatile("" ::: /* pretend to clobber */ "memory"); - RDTSC_START(cycles_start); - avxunpack(buffer, backdata,bit); - RDTSC_FINAL(cycles_final); - cycles_diff = (cycles_final - cycles_start); - if (cycles_diff < min_diff) min_diff = cycles_diff; - } - printf("%.2f\t",min_diff*1.0/length); - - free(data); - free(buffer); - free(backdata); - printf("\n"); - } - printf("\n\n"); /* two blank lines are required by gnuplot */ -} -#endif /* avx 2 */ - - -int main() { - demo128(); - demo128_d1(); -#ifdef __AVX2__ - demo256(); -#endif - return 0; - - -} diff --git a/cpp/simdcomp/example.c b/cpp/simdcomp/example.c deleted file mode 100644 index 1d68f95a7..000000000 --- a/cpp/simdcomp/example.c +++ /dev/null @@ -1,195 +0,0 @@ -/* Type "make example" to build this example program. */ -#include -#include -#include -#include "simdcomp.h" - -/** -We provide several different code examples. -**/ - - -/* very simple test to illustrate a simple application */ -int compress_decompress_demo() { - size_t k, N = 9999; - __m128i * endofbuf; - int howmanybytes; - float compratio; - uint32_t * datain = malloc(N * sizeof(uint32_t)); - uint8_t * buffer; - uint32_t * backbuffer = malloc(N * sizeof(uint32_t)); - uint32_t b; - printf("== simple test\n"); - - for (k = 0; k < N; ++k) { /* start with k=0, not k=1! */ - datain[k] = k; - } - - b = maxbits_length(datain, N); - buffer = malloc(simdpack_compressedbytes(N,b)); - endofbuf = simdpack_length(datain, N, (__m128i *)buffer, b); - howmanybytes = (endofbuf-(__m128i *)buffer)*sizeof(__m128i); /* number of compressed bytes */ - compratio = N*sizeof(uint32_t) * 1.0 / howmanybytes; - /* endofbuf points to the end of the compressed data */ - buffer = realloc(buffer,(endofbuf-(__m128i *)buffer)*sizeof(__m128i)); /* optional but safe. */ - printf("Compressed %d integers down to %d bytes (comp. ratio = %f).\n",(int)N,howmanybytes,compratio); - /* in actual applications b must be stored and retrieved: caller is responsible for that. */ - simdunpack_length((const __m128i *)buffer, N, backbuffer, b); /* will return a pointer to endofbuf */ - - for (k = 0; k < N; ++k) { - if(datain[k] != backbuffer[k]) { - printf("bug at %lu \n",(unsigned long)k); - return -1; - } - } - printf("Code works!\n"); - free(datain); - free(buffer); - free(backbuffer); - return 0; -} - - - -/* compresses data from datain to buffer, returns how many bytes written -used below in simple_demo */ -size_t compress(uint32_t * datain, size_t length, uint8_t * buffer) { - uint32_t offset; - uint8_t * initout; - size_t k; - if(length/SIMDBlockSize*SIMDBlockSize != length) { - printf("Data length should be a multiple of %i \n",SIMDBlockSize); - } - offset = 0; - initout = buffer; - for(k = 0; k < length / SIMDBlockSize; ++k) { - uint32_t b = simdmaxbitsd1(offset, - datain + k * SIMDBlockSize); - *buffer++ = b; - simdpackwithoutmaskd1(offset, datain + k * SIMDBlockSize, (__m128i *) buffer, - b); - offset = datain[k * SIMDBlockSize + SIMDBlockSize - 1]; - buffer += b * sizeof(__m128i); - } - return buffer - initout; -} - -/* Another illustration ... */ -void simple_demo() { - size_t REPEAT = 10, gap; - size_t N = 1000 * SIMDBlockSize;/* SIMDBlockSize is 128 */ - uint32_t * datain = malloc(N * sizeof(uint32_t)); - size_t compsize; - clock_t start, end; - uint8_t * buffer = malloc(N * sizeof(uint32_t) + N / SIMDBlockSize); /* output buffer */ - uint32_t * backbuffer = malloc(SIMDBlockSize * sizeof(uint32_t)); - printf("== simple demo\n"); - for (gap = 1; gap <= 243; gap *= 3) { - size_t k, repeat; - uint32_t offset = 0; - uint32_t bogus = 0; - double numberofseconds; - - printf("\n"); - printf(" gap = %lu \n", (unsigned long) gap); - datain[0] = 0; - for (k = 1; k < N; ++k) - datain[k] = datain[k-1] + ( rand() % (gap + 1) ); - compsize = compress(datain,N,buffer); - printf("compression ratio = %f \n", (N * sizeof(uint32_t))/ (compsize * 1.0 )); - start = clock(); - for(repeat = 0; repeat < REPEAT; ++repeat) { - uint8_t * decbuffer = buffer; - for (k = 0; k * SIMDBlockSize < N; ++k) { - uint8_t b = *decbuffer++; - simdunpackd1(offset, (__m128i *) decbuffer, backbuffer, b); - /* do something here with backbuffer */ - bogus += backbuffer[3]; - decbuffer += b * sizeof(__m128i); - offset = backbuffer[SIMDBlockSize - 1]; - } - } - end = clock(); - numberofseconds = (end-start)/(double)CLOCKS_PER_SEC; - printf("decoding speed in million of integers per second %f \n",N*REPEAT/(numberofseconds*1000.0*1000.0)); - start = clock(); - for(repeat = 0; repeat < REPEAT; ++repeat) { - uint8_t * decbuffer = buffer; - for (k = 0; k * SIMDBlockSize < N; ++k) { - memcpy(backbuffer,decbuffer+k*SIMDBlockSize,SIMDBlockSize*sizeof(uint32_t)); - bogus += backbuffer[3] - backbuffer[100]; - } - } - end = clock(); - numberofseconds = (end-start)/(double)CLOCKS_PER_SEC; - printf("memcpy speed in million of integers per second %f \n",N*REPEAT/(numberofseconds*1000.0*1000.0)); - printf("ignore me %i \n",bogus); - printf("All tests are in CPU cache. Avoid out-of-cache decoding in applications.\n"); - } - free(buffer); - free(datain); - free(backbuffer); -} - -/* Used below in more_sophisticated_demo ... */ -size_t varying_bit_width_compress(uint32_t * datain, size_t length, uint8_t * buffer) { - uint8_t * initout; - size_t k; - if(length/SIMDBlockSize*SIMDBlockSize != length) { - printf("Data length should be a multiple of %i \n",SIMDBlockSize); - } - initout = buffer; - for(k = 0; k < length / SIMDBlockSize; ++k) { - uint32_t b = maxbits(datain); - *buffer++ = b; - simdpackwithoutmask(datain, (__m128i *)buffer, b); - datain += SIMDBlockSize; - buffer += b * sizeof(__m128i); - } - return buffer - initout; -} - -/* Here we compress the data in blocks of 128 integers with varying bit width */ -int varying_bit_width_demo() { - size_t nn = 128 * 2; - uint32_t * datainn = malloc(nn * sizeof(uint32_t)); - uint8_t * buffern = malloc(nn * sizeof(uint32_t) + nn / SIMDBlockSize); - uint8_t * initbuffern = buffern; - uint32_t * backbuffern = malloc(nn * sizeof(uint32_t)); - size_t k, compsize; - printf("== varying bit-width demo\n"); - - for(k=0; k -*/ -import "C" -import "fmt" - -////////// -// For this demo, we pack and unpack blocks of 128 integers -///////// -func main() { - // I am going to use C types. Alternative might be to use unsafe.Pointer calls, see http://bit.ly/1ndw3W3 - // this is our original data - var data [128]C.uint32_t - for i := C.uint32_t(0); i < C.uint32_t(128); i++ { - data[i] = i - } - - - - - - //////////// - // We first pack without differential coding - /////////// - // computing how many bits per int. is needed - b := C.maxbits(&data[0]) - ratio := 32.0/float64(b) - fmt.Println("Bit width ", b) - fmt.Println(fmt.Sprintf("Compression ratio %f ", ratio)) - // we are now going to create a buffer to receive the packed data (each __m128i uses 128 bits) - out := make([] C.__m128i,b) - C.simdpackwithoutmask( &data[0],&out[0],b); - var recovereddata [128]C.uint32_t - C.simdunpack(&out[0],&recovereddata[0],b) - for i := 0; i < 128; i++ { - if data[i] != recovereddata[i] { - fmt.Println("Bug ") - return - } - } - - /////////// - // Next, we use differential coding - ////////// - offset := C.uint32_t(0) // if you pack data from K to K + 128, offset should be the value at K-1. When K = 0, choose a default - b1 := C.simdmaxbitsd1(offset,&data[0]) - ratio1 := 32.0/float64(b1) - fmt.Println("Bit width ", b1) - fmt.Println(fmt.Sprintf("Compression ratio %f ", ratio1)) - // we are now going to create a buffer to receive the packed data (each __m128i uses 128 bits) - out = make([] C.__m128i,b1) - C.simdpackwithoutmaskd1(offset, &data[0],&out[0],b1); - C.simdunpackd1(offset,&out[0],&recovereddata[0],b1) - for i := 0; i < 128; i++ { - if data[i] != recovereddata[i] { - fmt.Println("Bug ") - return - } - } - - fmt.Println("test succesful.") - -} diff --git a/cpp/simdcomp/include/avxbitpacking.h b/cpp/simdcomp/include/avxbitpacking.h deleted file mode 100644 index 00b68d622..000000000 --- a/cpp/simdcomp/include/avxbitpacking.h +++ /dev/null @@ -1,40 +0,0 @@ -/** - * This code is released under a BSD License. - */ - -#ifndef INCLUDE_AVXBITPACKING_H_ -#define INCLUDE_AVXBITPACKING_H_ - - -#ifdef __AVX2__ - -#include "portability.h" - - -/* AVX2 is required */ -#include -/* for memset */ -#include - -#include "simdcomputil.h" - -enum{ AVXBlockSize = 256}; - -/* max integer logarithm over a range of AVXBlockSize integers (256 integer) */ -uint32_t avxmaxbits(const uint32_t * begin); - -/* reads 256 values from "in", writes "bit" 256-bit vectors to "out" */ -void avxpack(const uint32_t * in,__m256i * out, const uint32_t bit); - -/* reads 256 values from "in", writes "bit" 256-bit vectors to "out" */ -void avxpackwithoutmask(const uint32_t * in,__m256i * out, const uint32_t bit); - -/* reads "bit" 256-bit vectors from "in", writes 256 values to "out" */ -void avxunpack(const __m256i * in,uint32_t * out, const uint32_t bit); - - - - -#endif /* __AVX2__ */ - -#endif /* INCLUDE_AVXBITPACKING_H_ */ diff --git a/cpp/simdcomp/include/portability.h b/cpp/simdcomp/include/portability.h deleted file mode 100644 index f68f179f1..000000000 --- a/cpp/simdcomp/include/portability.h +++ /dev/null @@ -1,81 +0,0 @@ -/** - * This code is released under a BSD License. - */ -#ifndef SIMDBITCOMPAT_H_ -#define SIMDBITCOMPAT_H_ - -#include /* mostly for Microsoft compilers */ -#include - -#if SIMDCOMP_DEBUG -# define SIMDCOMP_ALWAYS_INLINE inline -# define SIMDCOMP_NEVER_INLINE -# define SIMDCOMP_PURE -#else -# if defined(__GNUC__) -# if __GNUC__ >= 3 -# define SIMDCOMP_ALWAYS_INLINE inline __attribute__((always_inline)) -# define SIMDCOMP_NEVER_INLINE __attribute__((noinline)) -# define SIMDCOMP_PURE __attribute__((pure)) -# else -# define SIMDCOMP_ALWAYS_INLINE inline -# define SIMDCOMP_NEVER_INLINE -# define SIMDCOMP_PURE -# endif -# elif defined(_MSC_VER) -# define SIMDCOMP_ALWAYS_INLINE __forceinline -# define SIMDCOMP_NEVER_INLINE -# define SIMDCOMP_PURE -# else -# if __has_attribute(always_inline) -# define SIMDCOMP_ALWAYS_INLINE inline __attribute__((always_inline)) -# else -# define SIMDCOMP_ALWAYS_INLINE inline -# endif -# if __has_attribute(noinline) -# define SIMDCOMP_NEVER_INLINE __attribute__((noinline)) -# else -# define SIMDCOMP_NEVER_INLINE -# endif -# if __has_attribute(pure) -# define SIMDCOMP_PURE __attribute__((pure)) -# else -# define SIMDCOMP_PURE -# endif -# endif -#endif - -#if defined(_MSC_VER) && _MSC_VER < 1600 -typedef unsigned int uint32_t; -typedef unsigned char uint8_t; -typedef signed char int8_t; -#else -#include /* part of Visual Studio 2010 and better, others likely anyway */ -#endif - -#if defined(_MSC_VER) -#define SIMDCOMP_ALIGNED(x) __declspec(align(x)) -#else -#if defined(__GNUC__) -#define SIMDCOMP_ALIGNED(x) __attribute__ ((aligned(x))) -#endif -#endif - -#if defined(_MSC_VER) -# include -/* 64-bit needs extending */ -# define SIMDCOMP_CTZ(result, mask) do { \ - unsigned long index; \ - if (!_BitScanForward(&(index), (mask))) { \ - (result) = 32U; \ - } else { \ - (result) = (uint32_t)(index); \ - } \ - } while (0) -#else -# define SIMDCOMP_CTZ(result, mask) \ - result = __builtin_ctz(mask) -#endif - -#endif /* SIMDBITCOMPAT_H_ */ - diff --git a/cpp/simdcomp/include/simdbitpacking.h b/cpp/simdcomp/include/simdbitpacking.h deleted file mode 100644 index af32fd617..000000000 --- a/cpp/simdcomp/include/simdbitpacking.h +++ /dev/null @@ -1,72 +0,0 @@ -/** - * This code is released under a BSD License. - */ -#ifndef SIMDBITPACKING_H_ -#define SIMDBITPACKING_H_ - -#include "portability.h" - -/* SSE2 is required */ -#include -/* for memset */ -#include - -#include "simdcomputil.h" - -/*** -* Please see example.c for various examples on how to make good use -* of these functions. -*/ - - - -/* reads 128 values from "in", writes "bit" 128-bit vectors to "out". - * The input values are masked so that only the least significant "bit" bits are used. */ -void simdpack(const uint32_t * in,__m128i * out, const uint32_t bit); - -/* reads 128 values from "in", writes "bit" 128-bit vectors to "out". - * The input values are assumed to be less than 1< - - - - -/* returns the integer logarithm of v (bit width) */ -uint32_t bits(const uint32_t v); - -/* max integer logarithm over a range of SIMDBlockSize integers (128 integer) */ -uint32_t maxbits(const uint32_t * begin); - -/* same as maxbits, but we specify the number of integers */ -uint32_t maxbits_length(const uint32_t * in,uint32_t length); - -enum{ SIMDBlockSize = 128}; - - -/* computes (quickly) the minimal value of 128 values */ -uint32_t simdmin(const uint32_t * in); - -/* computes (quickly) the minimal value of the specified number of values */ -uint32_t simdmin_length(const uint32_t * in, uint32_t length); - -#ifdef __SSE4_1__ -/* computes (quickly) the minimal and maximal value of the specified number of values */ -void simdmaxmin_length(const uint32_t * in, uint32_t length, uint32_t * getmin, uint32_t * getmax); - -/* computes (quickly) the minimal and maximal value of the 128 values */ -void simdmaxmin(const uint32_t * in, uint32_t * getmin, uint32_t * getmax); - -#endif - -/* like maxbit over 128 integers (SIMDBlockSize) with provided initial value - and using differential coding */ -uint32_t simdmaxbitsd1(uint32_t initvalue, const uint32_t * in); - -/* like simdmaxbitsd1, but calculates maxbits over |length| integers - with provided initial value. |length| can be any arbitrary value. */ -uint32_t simdmaxbitsd1_length(uint32_t initvalue, const uint32_t * in, - uint32_t length); - - - -#endif /* SIMDCOMPUTIL_H_ */ diff --git a/cpp/simdcomp/include/simdfor.h b/cpp/simdcomp/include/simdfor.h deleted file mode 100644 index 39f7fbcac..000000000 --- a/cpp/simdcomp/include/simdfor.h +++ /dev/null @@ -1,72 +0,0 @@ -/** - * This code is released under a BSD License. - */ -#ifndef INCLUDE_SIMDFOR_H_ -#define INCLUDE_SIMDFOR_H_ - -#include "portability.h" - -/* SSE2 is required */ -#include - -#include "simdcomputil.h" -#include "simdbitpacking.h" - -#ifdef __cplusplus -extern "C" { -#endif - -/* reads 128 values from "in", writes "bit" 128-bit vectors to "out" */ -void simdpackFOR(uint32_t initvalue, const uint32_t * in,__m128i * out, const uint32_t bit); - - -/* reads "bit" 128-bit vectors from "in", writes 128 values to "out" */ -void simdunpackFOR(uint32_t initvalue, const __m128i * in,uint32_t * out, const uint32_t bit); - - -/* how many compressed bytes are needed to compressed length integers using a bit width of bit with -the simdpackFOR_length function. */ -int simdpackFOR_compressedbytes(int length, const uint32_t bit); - -/* like simdpackFOR, but supports an undetermined number of inputs. -This is useful if you need to pack less than 128 integers. Note that this function is much slower. - Compressed data is stored in the memory location between - the provided (out) pointer and the returned pointer. */ -__m128i * simdpackFOR_length(uint32_t initvalue, const uint32_t * in, int length, __m128i * out, const uint32_t bit); - -/* like simdunpackFOR, but supports an undetermined number of inputs. -This is useful if you need to unpack less than 128 integers. Note that this function is much slower. - The read compressed data is between the provided - (in) pointer and the returned pointer. */ -const __m128i * simdunpackFOR_length(uint32_t initvalue, const __m128i * in, int length, uint32_t * out, const uint32_t bit); - - -/* returns the value stored at the specified "slot". -* */ -uint32_t simdselectFOR(uint32_t initvalue, const __m128i *in, uint32_t bit, - int slot); - -/* given a block of 128 packed values, this function sets the value at index "index" to "value" */ -void simdfastsetFOR(uint32_t initvalue, __m128i * in, uint32_t bit, uint32_t value, size_t index); - - -/* searches "bit" 128-bit vectors from "in" (= length<=128 encoded integers) for the first encoded uint32 value - * which is >= |key|, and returns its position. It is assumed that the values - * stored are in sorted order. - * The encoded key is stored in "*presult". - * The first length decoded integers, ignoring others. If no value is larger or equal to the key, - * length is returned. Length should be no larger than 128. - * - * If no value is larger or equal to the key, -* length is returned */ -int simdsearchwithlengthFOR(uint32_t initvalue, const __m128i *in, uint32_t bit, - int length, uint32_t key, uint32_t *presult); - -#ifdef __cplusplus -} // extern "C" -#endif - - - - -#endif /* INCLUDE_SIMDFOR_H_ */ diff --git a/cpp/simdcomp/include/simdintegratedbitpacking.h b/cpp/simdcomp/include/simdintegratedbitpacking.h deleted file mode 100644 index 92f44a23e..000000000 --- a/cpp/simdcomp/include/simdintegratedbitpacking.h +++ /dev/null @@ -1,98 +0,0 @@ -/** - * This code is released under a BSD License. - */ - -#ifndef SIMD_INTEGRATED_BITPACKING_H -#define SIMD_INTEGRATED_BITPACKING_H - -#include "portability.h" - -/* SSE2 is required */ -#include - -#include "simdcomputil.h" -#include "simdbitpacking.h" - -#ifdef __cplusplus -extern "C" { -#endif - -/* reads 128 values from "in", writes "bit" 128-bit vectors to "out" - integer values should be in sorted order (for best results). - The differences are masked so that only the least significant "bit" bits are used. */ -void simdpackd1(uint32_t initvalue, const uint32_t * in,__m128i * out, const uint32_t bit); - - -/* reads 128 values from "in", writes "bit" 128-bit vectors to "out" - integer values should be in sorted order (for best results). - The difference values are assumed to be less than 1<= |key|, and returns its position. It is assumed that the values - * stored are in sorted order. - * The encoded key is stored in "*presult". If no value is larger or equal to the key, -* 128 is returned. The pointer initOffset is a pointer to the last four value decoded -* (when starting out, this can be a zero vector or initialized with _mm_set1_epi32(init)), -* and the vector gets updated. -**/ -int -simdsearchd1(__m128i * initOffset, const __m128i *in, uint32_t bit, - uint32_t key, uint32_t *presult); - - -/* searches "bit" 128-bit vectors from "in" (= length<=128 encoded integers) for the first encoded uint32 value - * which is >= |key|, and returns its position. It is assumed that the values - * stored are in sorted order. - * The encoded key is stored in "*presult". - * The first length decoded integers, ignoring others. If no value is larger or equal to the key, - * length is returned. Length should be no larger than 128. - * - * If no value is larger or equal to the key, -* length is returned */ -int simdsearchwithlengthd1(uint32_t initvalue, const __m128i *in, uint32_t bit, - int length, uint32_t key, uint32_t *presult); - - - -/* returns the value stored at the specified "slot". -* */ -uint32_t simdselectd1(uint32_t initvalue, const __m128i *in, uint32_t bit, - int slot); - -/* given a block of 128 packed values, this function sets the value at index "index" to "value", - * you must somehow know the previous value. - * Because of differential coding, all following values are incremented by the offset between this new - * value and the old value... - * This functions is useful if you want to modify the last value. - */ -void simdfastsetd1fromprevious( __m128i * in, uint32_t bit, uint32_t previousvalue, uint32_t value, size_t index); - -/* given a block of 128 packed values, this function sets the value at index "index" to "value", - * This function computes the previous value if needed. - * Because of differential coding, all following values are incremented by the offset between this new - * value and the old value... - * This functions is useful if you want to modify the last value. - */ -void simdfastsetd1(uint32_t initvalue, __m128i * in, uint32_t bit, uint32_t value, size_t index); - - -/*Simply scan the data -* The pointer initOffset is a pointer to the last four value decoded -* (when starting out, this can be a zero vector or initialized with _mm_set1_epi32(init);), -* and the vector gets updated. -* */ - -void -simdscand1(__m128i * initOffset, const __m128i *in, uint32_t bit); - -#ifdef __cplusplus -} // extern "C" -#endif - -#endif diff --git a/cpp/simdcomp/makefile b/cpp/simdcomp/makefile deleted file mode 100644 index b022de413..000000000 --- a/cpp/simdcomp/makefile +++ /dev/null @@ -1,79 +0,0 @@ -# minimalist makefile -.SUFFIXES: -# -.SUFFIXES: .cpp .o .c .h -ifeq ($(DEBUG),1) -CFLAGS = -fPIC -std=c89 -ggdb -msse4.1 -march=native -Wall -Wextra -Wshadow -fsanitize=undefined -fno-omit-frame-pointer -fsanitize=address -else -CFLAGS = -fPIC -std=c89 -O3 -msse4.1 -march=native -Wall -Wextra -Wshadow -endif # debug -LDFLAGS = -shared -LIBNAME=libsimdcomp.so.0.0.3 -all: unit unit_chars bitpackingbenchmark $(LIBNAME) -test: - ./unit - ./unit_chars -install: $(OBJECTS) - cp $(LIBNAME) /usr/local/lib - ln -s /usr/local/lib/$(LIBNAME) /usr/local/lib/libsimdcomp.so - ldconfig - cp $(HEADERS) /usr/local/include - - - -HEADERS=./include/simdbitpacking.h ./include/simdcomputil.h ./include/simdintegratedbitpacking.h ./include/simdcomp.h ./include/simdfor.h ./include/avxbitpacking.h - -uninstall: - for h in $(HEADERS) ; do rm /usr/local/$$h; done - rm /usr/local/lib/$(LIBNAME) - rm /usr/local/lib/libsimdcomp.so - ldconfig - - -OBJECTS= simdbitpacking.o simdintegratedbitpacking.o simdcomputil.o \ - simdpackedsearch.o simdpackedselect.o simdfor.o avxbitpacking.o - -$(LIBNAME): $(OBJECTS) - $(CC) $(CFLAGS) -o $(LIBNAME) $(OBJECTS) $(LDFLAGS) - - -avxbitpacking.o: ./src/avxbitpacking.c $(HEADERS) - $(CC) $(CFLAGS) -c ./src/avxbitpacking.c -Iinclude - - -simdfor.o: ./src/simdfor.c $(HEADERS) - $(CC) $(CFLAGS) -c ./src/simdfor.c -Iinclude - - -simdcomputil.o: ./src/simdcomputil.c $(HEADERS) - $(CC) $(CFLAGS) -c ./src/simdcomputil.c -Iinclude - -simdbitpacking.o: ./src/simdbitpacking.c $(HEADERS) - $(CC) $(CFLAGS) -c ./src/simdbitpacking.c -Iinclude - -simdintegratedbitpacking.o: ./src/simdintegratedbitpacking.c $(HEADERS) - $(CC) $(CFLAGS) -c ./src/simdintegratedbitpacking.c -Iinclude - -simdpackedsearch.o: ./src/simdpackedsearch.c $(HEADERS) - $(CC) $(CFLAGS) -c ./src/simdpackedsearch.c -Iinclude - -simdpackedselect.o: ./src/simdpackedselect.c $(HEADERS) - $(CC) $(CFLAGS) -c ./src/simdpackedselect.c -Iinclude - -example: ./example.c $(HEADERS) $(OBJECTS) - $(CC) $(CFLAGS) -o example ./example.c -Iinclude $(OBJECTS) - -unit: ./tests/unit.c $(HEADERS) $(OBJECTS) - $(CC) $(CFLAGS) -o unit ./tests/unit.c -Iinclude $(OBJECTS) - -bitpackingbenchmark: ./benchmarks/bitpackingbenchmark.c $(HEADERS) $(OBJECTS) - $(CC) $(CFLAGS) -o bitpackingbenchmark ./benchmarks/bitpackingbenchmark.c -Iinclude $(OBJECTS) -benchmark: ./benchmarks/benchmark.c $(HEADERS) $(OBJECTS) - $(CC) $(CFLAGS) -o benchmark ./benchmarks/benchmark.c -Iinclude $(OBJECTS) -dynunit: ./tests/unit.c $(HEADERS) $(LIBNAME) - $(CC) $(CFLAGS) -o dynunit ./tests/unit.c -Iinclude -lsimdcomp - -unit_chars: ./tests/unit_chars.c $(HEADERS) $(OBJECTS) - $(CC) $(CFLAGS) -o unit_chars ./tests/unit_chars.c -Iinclude $(OBJECTS) -clean: - rm -f unit *.o $(LIBNAME) example benchmark bitpackingbenchmark dynunit unit_chars diff --git a/cpp/simdcomp/makefile.vc b/cpp/simdcomp/makefile.vc deleted file mode 100644 index cb631c272..000000000 --- a/cpp/simdcomp/makefile.vc +++ /dev/null @@ -1,104 +0,0 @@ - -!IFNDEF MACHINE -!IF "$(PROCESSOR_ARCHITECTURE)"=="AMD64" -MACHINE=x64 -!ELSE -MACHINE=x86 -!ENDIF -!ENDIF - -!IFNDEF DEBUG -DEBUG=no -!ENDIF - -!IFNDEF CC -CC=cl.exe -!ENDIF - -!IFNDEF AR -AR=lib.exe -!ENDIF - -!IFNDEF LINK -LINK=link.exe -!ENDIF - -!IFNDEF PGO -PGO=no -!ENDIF - -!IFNDEF PGI -PGI=no -!ENDIF - -INC = /Iinclude - -!IF "$(DEBUG)"=="yes" -CFLAGS = /nologo /MDd /LDd /Od /Zi /D_DEBUG /RTC1 /W3 /GS /Gm -ARFLAGS = /nologo -LDFLAGS = /nologo /debug /nodefaultlib:msvcrt -!ELSE -CFLAGS = /nologo /MD /O2 /Zi /DNDEBUG /W3 /Gm- /GS /Gy /Oi /GL /MP -ARFLAGS = /nologo /LTCG -LDFLAGS = /nologo /LTCG /DYNAMICBASE /incremental:no /debug /opt:ref,icf -!ENDIF - -!IF "$(PGI)"=="yes" -LDFLAGS = $(LDFLAGS) /ltcg:pgi -!ENDIF - -!IF "$(PGO)"=="yes" -LDFLAGS = $(LDFLAGS) /ltcg:pgo -!ENDIF - -LIB_OBJS = simdbitpacking.obj simdintegratedbitpacking.obj simdcomputil.obj \ - simdpackedsearch.obj simdpackedselect.obj simdfor.obj - - -all: lib dll dynunit unit_chars example benchmark -# need some good use case scenario to train the instrumented build - @if "$(PGI)"=="yes" echo Running PGO training - @if "$(PGI)"=="yes" benchmark.exe >nul 2>&1 - @if "$(PGI)"=="yes" example.exe >nul 2>&1 - - -$(LIB_OBJS): - $(CC) $(INC) $(CFLAGS) /c src/simdbitpacking.c src/simdintegratedbitpacking.c src/simdcomputil.c \ - src/simdpackedsearch.c src/simdpackedselect.c src/simdfor.c - -lib: $(LIB_OBJS) - $(AR) $(ARFLAGS) /OUT:simdcomp_a.lib $(LIB_OBJS) - -dll: $(LIB_OBJS) - $(LINK) /DLL $(LDFLAGS) /OUT:simdcomp.dll /IMPLIB:simdcomp.lib /DEF:simdcomp.def $(LIB_OBJS) - -unit: lib - $(CC) $(INC) $(CFLAGS) /c src/unit.c - $(LINK) $(LDFLAGS) /OUT:unit.exe unit.obj simdcomp_a.lib - -dynunit: dll - $(CC) $(INC) $(CFLAGS) /c src/unit.c - $(LINK) $(LDFLAGS) /OUT:unit.exe unit.obj simdcomp.lib - -unit_chars: lib - $(CC) $(INC) $(CFLAGS) /c src/unit_chars.c - $(LINK) $(LDFLAGS) /OUT:unit_chars.exe unit_chars.obj simdcomp.lib - - -example: lib - $(CC) $(INC) $(CFLAGS) /c example.c - $(LINK) $(LDFLAGS) /OUT:example.exe example.obj simdcomp.lib - -benchmark: lib - $(CC) $(INC) $(CFLAGS) /c src/benchmark.c - $(LINK) $(LDFLAGS) /OUT:benchmark.exe benchmark.obj simdcomp.lib - -clean: - del /Q *.obj - del /Q *.lib - del /Q *.exe - del /Q *.dll - del /Q *.pgc - del /Q *.pgd - del /Q *.pdb - diff --git a/cpp/simdcomp/package.json b/cpp/simdcomp/package.json deleted file mode 100644 index a91dd24a4..000000000 --- a/cpp/simdcomp/package.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "name": "simdcomp", - "version": "0.0.3", - "repo": "lemire/simdcomp", - "description": "A simple C library for compressing lists of integers", - "license": "BSD-3-Clause", - "src": [ - "src/simdbitpacking.c", - "src/simdcomputil.c", - "src/simdintegratedbitpacking.c", - "include/simdbitpacking.h", - "include/simdcomp.h", - "include/simdcomputil.h", - "include/simdintegratedbitpacking.h" - ] -} diff --git a/cpp/simdcomp/scripts/avxpacking.py b/cpp/simdcomp/scripts/avxpacking.py deleted file mode 100755 index 81d1ac097..000000000 --- a/cpp/simdcomp/scripts/avxpacking.py +++ /dev/null @@ -1,182 +0,0 @@ -#!/usr/bin/env python -import sys -def howmany(bit): - """ how many values are we going to pack? """ - return 256 - -def howmanywords(bit): - return (howmany(bit) * bit + 255)/256 - -def howmanybytes(bit): - return howmanywords(bit) * 16 - -print(""" -/** code generated by avxpacking.py starts here **/ -""") - -print("""typedef void (*avxpackblockfnc)(const uint32_t * pin, __m256i * compressed);""") -print("""typedef void (*avxunpackblockfnc)(const __m256i * compressed, uint32_t * pout);""") - - - - - - -def plurial(number): - if(number <> 1): - return "s" - else : - return "" - -print("") -print("static void avxpackblock0(const uint32_t * pin, __m256i * compressed) {"); -print(" (void)compressed;"); -print(" (void) pin; /* we consumed {0} 32-bit integer{1} */ ".format(howmany(0),plurial(howmany(0)))); -print("}"); -print("") - -for bit in range(1,33): - print("") - print("/* we are going to pack {0} {1}-bit values, touching {2} 256-bit words, using {3} bytes */ ".format(howmany(bit),bit,howmanywords(bit),howmanybytes(bit))) - print("static void avxpackblock{0}(const uint32_t * pin, __m256i * compressed) {{".format(bit)); - print(" const __m256i * in = (const __m256i *) pin;"); - print(" /* we are going to touch {0} 256-bit word{1} */ ".format(howmanywords(bit),plurial(howmanywords(bit)))); - if(howmanywords(bit) == 1): - print(" __m256i w0;") - else: - print(" __m256i w0, w1;") - if( (bit & (bit-1)) <> 0) : print(" __m256i tmp; /* used to store inputs at word boundary */") - oldword = 0 - for j in range(howmany(bit)/8): - firstword = j * bit / 32 - if(firstword > oldword): - print(" _mm256_storeu_si256(compressed + {0}, w{1});".format(oldword,oldword%2)) - oldword = firstword - secondword = (j * bit + bit - 1)/32 - firstshift = (j*bit) % 32 - if( firstword == secondword): - if(firstshift == 0): - print(" w{0} = _mm256_lddqu_si256 (in + {1});".format(firstword%2,j)) - else: - print(" w{0} = _mm256_or_si256(w{0},_mm256_slli_epi32(_mm256_lddqu_si256 (in + {1}) , {2}));".format(firstword%2,j,firstshift)) - else: - print(" tmp = _mm256_lddqu_si256 (in + {0});".format(j)) - print(" w{0} = _mm256_or_si256(w{0},_mm256_slli_epi32(tmp , {2}));".format(firstword%2,j,firstshift)) - secondshift = 32-firstshift - print(" w{0} = _mm256_srli_epi32(tmp,{2});".format(secondword%2,j,secondshift)) - print(" _mm256_storeu_si256(compressed + {0}, w{1});".format(secondword,secondword%2)) - print("}"); - print("") - - -print("") -print("static void avxpackblockmask0(const uint32_t * pin, __m256i * compressed) {"); -print(" (void)compressed;"); -print(" (void) pin; /* we consumed {0} 32-bit integer{1} */ ".format(howmany(0),plurial(howmany(0)))); -print("}"); -print("") - -for bit in range(1,33): - print("") - print("/* we are going to pack {0} {1}-bit values, touching {2} 256-bit words, using {3} bytes */ ".format(howmany(bit),bit,howmanywords(bit),howmanybytes(bit))) - print("static void avxpackblockmask{0}(const uint32_t * pin, __m256i * compressed) {{".format(bit)); - print(" /* we are going to touch {0} 256-bit word{1} */ ".format(howmanywords(bit),plurial(howmanywords(bit)))); - if(howmanywords(bit) == 1): - print(" __m256i w0;") - else: - print(" __m256i w0, w1;") - print(" const __m256i * in = (const __m256i *) pin;"); - if(bit < 32): print(" const __m256i mask = _mm256_set1_epi32({0});".format((1< 0) : print(" __m256i tmp; /* used to store inputs at word boundary */") - oldword = 0 - for j in range(howmany(bit)/8): - firstword = j * bit / 32 - if(firstword > oldword): - print(" _mm256_storeu_si256(compressed + {0}, w{1});".format(oldword,oldword%2)) - oldword = firstword - secondword = (j * bit + bit - 1)/32 - firstshift = (j*bit) % 32 - loadstr = maskfnc(" _mm256_lddqu_si256 (in + {0}) ".format(j)) - if( firstword == secondword): - if(firstshift == 0): - print(" w{0} = {1};".format(firstword%2,loadstr)) - else: - print(" w{0} = _mm256_or_si256(w{0},_mm256_slli_epi32({1} , {2}));".format(firstword%2,loadstr,firstshift)) - else: - print(" tmp = {0};".format(loadstr)) - print(" w{0} = _mm256_or_si256(w{0},_mm256_slli_epi32(tmp , {2}));".format(firstword%2,j,firstshift)) - secondshift = 32-firstshift - print(" w{0} = _mm256_srli_epi32(tmp,{2});".format(secondword%2,j,secondshift)) - print(" _mm256_storeu_si256(compressed + {0}, w{1});".format(secondword,secondword%2)) - print("}"); - print("") - - -print("static void avxunpackblock0(const __m256i * compressed, uint32_t * pout) {"); -print(" (void) compressed;"); -print(" memset(pout,0,{0});".format(howmany(0))); -print("}"); -print("") - -for bit in range(1,33): - print("") - print("/* we packed {0} {1}-bit values, touching {2} 256-bit words, using {3} bytes */ ".format(howmany(bit),bit,howmanywords(bit),howmanybytes(bit))) - print("static void avxunpackblock{0}(const __m256i * compressed, uint32_t * pout) {{".format(bit)); - print(" /* we are going to access {0} 256-bit word{1} */ ".format(howmanywords(bit),plurial(howmanywords(bit)))); - if(howmanywords(bit) == 1): - print(" __m256i w0;") - else: - print(" __m256i w0, w1;") - print(" __m256i * out = (__m256i *) pout;"); - if(bit < 32): print(" const __m256i mask = _mm256_set1_epi32({0});".format((1< oldword): - print(" w{0} = _mm256_lddqu_si256 (compressed + {1});".format(secondword%2,secondword)) - oldword = secondword - firstshift = (j*bit) % 32 - firstshiftstr = "_mm256_srli_epi32( w{0} , "+str(firstshift)+") " - if(firstshift == 0): - firstshiftstr =" w{0} " # no need - wfirst = firstshiftstr.format(firstword%2) - if( firstword == secondword): - if(firstshift + bit <> 32): - wfirst = maskstr.format(wfirst) - print(" _mm256_storeu_si256(out + {0}, {1});".format(j,wfirst)) - else: - secondshift = (32-firstshift) - wsecond = "_mm256_slli_epi32( w{0} , {1} ) ".format((firstword+1)%2,secondshift) - wfirstorsecond = " _mm256_or_si256 ({0},{1}) ".format(wfirst,wsecond) - wfirstorsecond = maskstr.format(wfirstorsecond) - print(" _mm256_storeu_si256(out + {0},\n {1});".format(j,wfirstorsecond)) - print("}"); - print("") - - -print("static avxpackblockfnc avxfuncPackArr[] = {") -for bit in range(0,32): - print("&avxpackblock{0},".format(bit)) -print("&avxpackblock32") -print("};") - -print("static avxpackblockfnc avxfuncPackMaskArr[] = {") -for bit in range(0,32): - print("&avxpackblockmask{0},".format(bit)) -print("&avxpackblockmask32") -print("};") - - -print("static avxunpackblockfnc avxfuncUnpackArr[] = {") -for bit in range(0,32): - print("&avxunpackblock{0},".format(bit)) -print("&avxunpackblock32") -print("};") -print("/** code generated by avxpacking.py ends here **/") diff --git a/cpp/simdcomp/scripts/simdfor.py b/cpp/simdcomp/scripts/simdfor.py deleted file mode 100755 index c60db1e89..000000000 --- a/cpp/simdcomp/scripts/simdfor.py +++ /dev/null @@ -1,152 +0,0 @@ -#!/usr/bin/env python3 - - -from math import ceil - -print(""" -/** -* Blablabla -* -*/ - -"""); - -def mask(bit): - return str((1 << bit) - 1) - -for length in [32]: - print(""" -static __m128i iunpackFOR0(__m128i initOffset, const __m128i * _in , uint32_t * _out) { - __m128i *out = (__m128i*)(_out); - int i; - (void) _in; - for (i = 0; i < 8; ++i) { - _mm_store_si128(out++, initOffset); - _mm_store_si128(out++, initOffset); - _mm_store_si128(out++, initOffset); - _mm_store_si128(out++, initOffset); - } - - return initOffset; -} - - """) - print(""" - -static void ipackFOR0(__m128i initOffset , const uint32_t * _in , __m128i * out ) { - (void) initOffset; - (void) _in; - (void) out; -} -""") - for bit in range(1,33): - offsetVar = " initOffset"; - print(""" -static void ipackFOR"""+str(bit)+"""(__m128i """+offsetVar+""", const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - """); - - if (bit != 32): - print(" __m128i CurrIn = _mm_load_si128(in);"); - print(" __m128i InReg = _mm_sub_epi32(CurrIn, initOffset);"); - else: - print(" __m128i InReg = _mm_load_si128(in);"); - print(" (void) initOffset;"); - - - inwordpointer = 0 - valuecounter = 0 - for k in range(ceil((length * bit) / 32)): - if(valuecounter == length): break - for x in range(inwordpointer,32,bit): - if(x!=0) : - print(" OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, " + str(x) + "));"); - else: - print(" OutReg = InReg; "); - if((x+bit>=32) ): - while(inwordpointer<32): - inwordpointer += bit - print(" _mm_store_si128(out, OutReg);"); - print(""); - - if(valuecounter + 1 < length): - print(" ++out;") - inwordpointer -= 32; - if(inwordpointer>0): - print(" OutReg = _mm_srli_epi32(InReg, " + str(bit) + " - " + str(inwordpointer) + ");"); - if(valuecounter + 1 < length): - print(" ++in;") - - if (bit != 32): - print(" CurrIn = _mm_load_si128(in);"); - print(" InReg = _mm_sub_epi32(CurrIn, initOffset);"); - else: - print(" InReg = _mm_load_si128(in);"); - print(""); - valuecounter = valuecounter + 1 - if(valuecounter == length): break - assert(valuecounter == length) - print("\n}\n\n""") - - for bit in range(1,32): - offsetVar = " initOffset"; - print("""\n -static __m128i iunpackFOR"""+str(bit)+"""(__m128i """+offsetVar+""", const __m128i* in, uint32_t * _out) { - """); - print(""" __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_load_si128(in); - __m128i OutReg; - __m128i tmp; - const __m128i mask = _mm_set1_epi32((1U<<"""+str(bit)+""")-1); - - """); - - MainText = ""; - - MainText += "\n"; - inwordpointer = 0 - valuecounter = 0 - for k in range(ceil((length * bit) / 32)): - for x in range(inwordpointer,32,bit): - if(valuecounter == length): break - if (x > 0): - MainText += " tmp = _mm_srli_epi32(InReg," + str(x) +");\n"; - else: - MainText += " tmp = InReg;\n"; - if(x+bit<32): - MainText += " OutReg = _mm_and_si128(tmp, mask);\n"; - else: - MainText += " OutReg = tmp;\n"; - if((x+bit>=32) ): - while(inwordpointer<32): - inwordpointer += bit - if(valuecounter + 1 < length): - MainText += " ++in;" - MainText += " InReg = _mm_load_si128(in);\n"; - inwordpointer -= 32; - if(inwordpointer>0): - MainText += " OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, " + str(bit) + "-" + str(inwordpointer) + "), mask));\n\n"; - if (bit != 32): - MainText += " OutReg = _mm_add_epi32(OutReg, initOffset);\n"; - MainText += " _mm_store_si128(out++, OutReg);\n\n"; - MainText += ""; - valuecounter = valuecounter + 1 - if(valuecounter == length): break - assert(valuecounter == length) - print(MainText) - print(" return initOffset;"); - print("\n}\n\n") - print(""" -static __m128i iunpackFOR32(__m128i initvalue , const __m128i* in, uint32_t * _out) { - __m128i * mout = (__m128i *)_out; - __m128i invec; - size_t k; - for(k = 0; k < 128/4; ++k) { - invec = _mm_load_si128(in++); - _mm_store_si128(mout++, invec); - } - return invec; -} - """) diff --git a/cpp/simdcomp/simdcomp.def b/cpp/simdcomp/simdcomp.def deleted file mode 100644 index 343e148c9..000000000 --- a/cpp/simdcomp/simdcomp.def +++ /dev/null @@ -1,40 +0,0 @@ -EXPORTS - simdpack - simdpackwithoutmask - simdunpack - bits - maxbits - maxbits_length - simdmin - simdmin_length - simdmaxmin - simdmaxmin_length - simdmaxbitsd1 - simdmaxbitsd1_length - simdpackd1 - simdpackwithoutmaskd1 - simdunpackd1 - simdsearchd1 - simdsearchwithlengthd1 - simdselectd1 - simdpackFOR - simdselectFOR - simdsearchwithlengthFOR - simdunpackFOR - simdmin_length - simdmaxmin - simdmaxmin_length - simdpack_length - simdpackFOR_length - simdunpackFOR_length - simdpack_shortlength - simdfastsetFOR - simdfastset - simdfastsetd1 - simdunpack_length - simdunpack_shortlength - simdsearchwithlengthFOR - simdscand1 - simdfastsetd1fromprevious - simdfastsetd1 - diff --git a/cpp/simdcomp/src/avxbitpacking.c b/cpp/simdcomp/src/avxbitpacking.c deleted file mode 100644 index f592e3687..000000000 --- a/cpp/simdcomp/src/avxbitpacking.c +++ /dev/null @@ -1,7795 +0,0 @@ -#include "avxbitpacking.h" -#ifdef __AVX2__ - - -static uint32_t maxbitas32int(const __m256i accumulator) { - const __m256i _tmp1 = _mm256_or_si256(_mm256_srli_si256(accumulator, 8), accumulator); /* (A,B,C,D) xor (0,0,A,B) = (A,B,C xor A,D xor B)*/ - const __m256i _tmp2 = _mm256_or_si256(_mm256_srli_si256(_tmp1, 4), _tmp1); /* (A,B,C xor A,D xor B) xor (0,0,0,C xor A)*/ - uint32_t ans1 = _mm256_extract_epi32(_tmp2,0); - uint32_t ans2 = _mm256_extract_epi32(_tmp2,4); - uint32_t ans = ans1 > ans2 ? ans1 : ans2; - return bits(ans); -} - -uint32_t avxmaxbits(const uint32_t * begin) { - const __m256i* pin = (const __m256i*)(begin); - __m256i accumulator = _mm256_lddqu_si256(pin); - uint32_t k = 1; - for(; 8*k < AVXBlockSize; ++k) { - __m256i newvec = _mm256_lddqu_si256(pin+k); - accumulator = _mm256_or_si256(accumulator,newvec); - } - return maxbitas32int(accumulator); -} - - -/** code generated by avxpacking.py starts here **/ - -typedef void (*avxpackblockfnc)(const uint32_t * pin, __m256i * compressed); -typedef void (*avxunpackblockfnc)(const __m256i * compressed, uint32_t * pout); - -static void avxpackblock0(const uint32_t * pin, __m256i * compressed) { - (void)compressed; - (void) pin; /* we consumed 256 32-bit integers */ -} - - -/* we are going to pack 256 1-bit values, touching 1 256-bit words, using 16 bytes */ -static void avxpackblock1(const uint32_t * pin, __m256i * compressed) { - const __m256i * in = (const __m256i *) pin; - /* we are going to touch 1 256-bit word */ - __m256i w0; - w0 = _mm256_lddqu_si256 (in + 0); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 1) , 1)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 2) , 2)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 3) , 3)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 4) , 4)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 5) , 5)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 6) , 6)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 7) , 7)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 8) , 8)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 9) , 9)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 10) , 10)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 11) , 11)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 12) , 12)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 13) , 13)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 14) , 14)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 15) , 15)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 16) , 16)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 17) , 17)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 18) , 18)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 19) , 19)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 20) , 20)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 21) , 21)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 22) , 22)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 23) , 23)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 24) , 24)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 25) , 25)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 26) , 26)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 27) , 27)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 28) , 28)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 29) , 29)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 30) , 30)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 31) , 31)); - _mm256_storeu_si256(compressed + 0, w0); -} - - -/* we are going to pack 256 2-bit values, touching 2 256-bit words, using 32 bytes */ -static void avxpackblock2(const uint32_t * pin, __m256i * compressed) { - const __m256i * in = (const __m256i *) pin; - /* we are going to touch 2 256-bit words */ - __m256i w0, w1; - w0 = _mm256_lddqu_si256 (in + 0); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 1) , 2)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 2) , 4)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 3) , 6)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 4) , 8)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 5) , 10)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 6) , 12)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 7) , 14)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 8) , 16)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 9) , 18)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 10) , 20)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 11) , 22)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 12) , 24)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 13) , 26)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 14) , 28)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 15) , 30)); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_lddqu_si256 (in + 16); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 17) , 2)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 18) , 4)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 19) , 6)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 20) , 8)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 21) , 10)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 22) , 12)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 23) , 14)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 24) , 16)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 25) , 18)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 26) , 20)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 27) , 22)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 28) , 24)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 29) , 26)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 30) , 28)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 31) , 30)); - _mm256_storeu_si256(compressed + 1, w1); -} - - -/* we are going to pack 256 3-bit values, touching 3 256-bit words, using 48 bytes */ -static void avxpackblock3(const uint32_t * pin, __m256i * compressed) { - const __m256i * in = (const __m256i *) pin; - /* we are going to touch 3 256-bit words */ - __m256i w0, w1; - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_lddqu_si256 (in + 0); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 1) , 3)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 2) , 6)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 3) , 9)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 4) , 12)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 5) , 15)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 6) , 18)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 7) , 21)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 8) , 24)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 9) , 27)); - tmp = _mm256_lddqu_si256 (in + 10); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 30)); - w1 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 11) , 1)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 12) , 4)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 13) , 7)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 14) , 10)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 15) , 13)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 16) , 16)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 17) , 19)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 18) , 22)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 19) , 25)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 20) , 28)); - tmp = _mm256_lddqu_si256 (in + 21); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 31)); - w0 = _mm256_srli_epi32(tmp,1); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 22) , 2)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 23) , 5)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 24) , 8)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 25) , 11)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 26) , 14)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 27) , 17)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 28) , 20)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 29) , 23)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 30) , 26)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 31) , 29)); - _mm256_storeu_si256(compressed + 2, w0); -} - - -/* we are going to pack 256 4-bit values, touching 4 256-bit words, using 64 bytes */ -static void avxpackblock4(const uint32_t * pin, __m256i * compressed) { - const __m256i * in = (const __m256i *) pin; - /* we are going to touch 4 256-bit words */ - __m256i w0, w1; - w0 = _mm256_lddqu_si256 (in + 0); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 1) , 4)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 2) , 8)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 3) , 12)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 4) , 16)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 5) , 20)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 6) , 24)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 7) , 28)); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_lddqu_si256 (in + 8); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 9) , 4)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 10) , 8)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 11) , 12)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 12) , 16)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 13) , 20)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 14) , 24)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 15) , 28)); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_lddqu_si256 (in + 16); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 17) , 4)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 18) , 8)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 19) , 12)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 20) , 16)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 21) , 20)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 22) , 24)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 23) , 28)); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_lddqu_si256 (in + 24); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 25) , 4)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 26) , 8)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 27) , 12)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 28) , 16)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 29) , 20)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 30) , 24)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 31) , 28)); - _mm256_storeu_si256(compressed + 3, w1); -} - - -/* we are going to pack 256 5-bit values, touching 5 256-bit words, using 80 bytes */ -static void avxpackblock5(const uint32_t * pin, __m256i * compressed) { - const __m256i * in = (const __m256i *) pin; - /* we are going to touch 5 256-bit words */ - __m256i w0, w1; - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_lddqu_si256 (in + 0); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 1) , 5)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 2) , 10)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 3) , 15)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 4) , 20)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 5) , 25)); - tmp = _mm256_lddqu_si256 (in + 6); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 30)); - w1 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 7) , 3)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 8) , 8)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 9) , 13)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 10) , 18)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 11) , 23)); - tmp = _mm256_lddqu_si256 (in + 12); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 13) , 1)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 14) , 6)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 15) , 11)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 16) , 16)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 17) , 21)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 18) , 26)); - tmp = _mm256_lddqu_si256 (in + 19); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 31)); - w1 = _mm256_srli_epi32(tmp,1); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 20) , 4)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 21) , 9)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 22) , 14)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 23) , 19)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 24) , 24)); - tmp = _mm256_lddqu_si256 (in + 25); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 29)); - w0 = _mm256_srli_epi32(tmp,3); - _mm256_storeu_si256(compressed + 3, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 26) , 2)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 27) , 7)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 28) , 12)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 29) , 17)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 30) , 22)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 31) , 27)); - _mm256_storeu_si256(compressed + 4, w0); -} - - -/* we are going to pack 256 6-bit values, touching 6 256-bit words, using 96 bytes */ -static void avxpackblock6(const uint32_t * pin, __m256i * compressed) { - const __m256i * in = (const __m256i *) pin; - /* we are going to touch 6 256-bit words */ - __m256i w0, w1; - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_lddqu_si256 (in + 0); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 1) , 6)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 2) , 12)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 3) , 18)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 4) , 24)); - tmp = _mm256_lddqu_si256 (in + 5); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 30)); - w1 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 6) , 4)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 7) , 10)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 8) , 16)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 9) , 22)); - tmp = _mm256_lddqu_si256 (in + 10); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 11) , 2)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 12) , 8)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 13) , 14)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 14) , 20)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 15) , 26)); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_lddqu_si256 (in + 16); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 17) , 6)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 18) , 12)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 19) , 18)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 20) , 24)); - tmp = _mm256_lddqu_si256 (in + 21); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 30)); - w0 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 3, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 22) , 4)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 23) , 10)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 24) , 16)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 25) , 22)); - tmp = _mm256_lddqu_si256 (in + 26); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 27) , 2)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 28) , 8)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 29) , 14)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 30) , 20)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 31) , 26)); - _mm256_storeu_si256(compressed + 5, w1); -} - - -/* we are going to pack 256 7-bit values, touching 7 256-bit words, using 112 bytes */ -static void avxpackblock7(const uint32_t * pin, __m256i * compressed) { - const __m256i * in = (const __m256i *) pin; - /* we are going to touch 7 256-bit words */ - __m256i w0, w1; - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_lddqu_si256 (in + 0); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 1) , 7)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 2) , 14)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 3) , 21)); - tmp = _mm256_lddqu_si256 (in + 4); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 5) , 3)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 6) , 10)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 7) , 17)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 8) , 24)); - tmp = _mm256_lddqu_si256 (in + 9); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 31)); - w0 = _mm256_srli_epi32(tmp,1); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 10) , 6)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 11) , 13)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 12) , 20)); - tmp = _mm256_lddqu_si256 (in + 13); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 27)); - w1 = _mm256_srli_epi32(tmp,5); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 14) , 2)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 15) , 9)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 16) , 16)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 17) , 23)); - tmp = _mm256_lddqu_si256 (in + 18); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 30)); - w0 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 3, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 19) , 5)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 20) , 12)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 21) , 19)); - tmp = _mm256_lddqu_si256 (in + 22); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 26)); - w1 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 23) , 1)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 24) , 8)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 25) , 15)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 26) , 22)); - tmp = _mm256_lddqu_si256 (in + 27); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 29)); - w0 = _mm256_srli_epi32(tmp,3); - _mm256_storeu_si256(compressed + 5, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 28) , 4)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 29) , 11)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 30) , 18)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 31) , 25)); - _mm256_storeu_si256(compressed + 6, w0); -} - - -/* we are going to pack 256 8-bit values, touching 8 256-bit words, using 128 bytes */ -static void avxpackblock8(const uint32_t * pin, __m256i * compressed) { - const __m256i * in = (const __m256i *) pin; - /* we are going to touch 8 256-bit words */ - __m256i w0, w1; - w0 = _mm256_lddqu_si256 (in + 0); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 1) , 8)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 2) , 16)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 3) , 24)); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_lddqu_si256 (in + 4); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 5) , 8)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 6) , 16)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 7) , 24)); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_lddqu_si256 (in + 8); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 9) , 8)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 10) , 16)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 11) , 24)); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_lddqu_si256 (in + 12); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 13) , 8)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 14) , 16)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 15) , 24)); - _mm256_storeu_si256(compressed + 3, w1); - w0 = _mm256_lddqu_si256 (in + 16); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 17) , 8)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 18) , 16)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 19) , 24)); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_lddqu_si256 (in + 20); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 21) , 8)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 22) , 16)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 23) , 24)); - _mm256_storeu_si256(compressed + 5, w1); - w0 = _mm256_lddqu_si256 (in + 24); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 25) , 8)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 26) , 16)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 27) , 24)); - _mm256_storeu_si256(compressed + 6, w0); - w1 = _mm256_lddqu_si256 (in + 28); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 29) , 8)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 30) , 16)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 31) , 24)); - _mm256_storeu_si256(compressed + 7, w1); -} - - -/* we are going to pack 256 9-bit values, touching 9 256-bit words, using 144 bytes */ -static void avxpackblock9(const uint32_t * pin, __m256i * compressed) { - const __m256i * in = (const __m256i *) pin; - /* we are going to touch 9 256-bit words */ - __m256i w0, w1; - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_lddqu_si256 (in + 0); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 1) , 9)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 2) , 18)); - tmp = _mm256_lddqu_si256 (in + 3); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 27)); - w1 = _mm256_srli_epi32(tmp,5); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 4) , 4)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 5) , 13)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 6) , 22)); - tmp = _mm256_lddqu_si256 (in + 7); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 31)); - w0 = _mm256_srli_epi32(tmp,1); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 8) , 8)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 9) , 17)); - tmp = _mm256_lddqu_si256 (in + 10); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 26)); - w1 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 11) , 3)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 12) , 12)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 13) , 21)); - tmp = _mm256_lddqu_si256 (in + 14); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 30)); - w0 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 3, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 15) , 7)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 16) , 16)); - tmp = _mm256_lddqu_si256 (in + 17); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 25)); - w1 = _mm256_srli_epi32(tmp,7); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 18) , 2)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 19) , 11)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 20) , 20)); - tmp = _mm256_lddqu_si256 (in + 21); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 29)); - w0 = _mm256_srli_epi32(tmp,3); - _mm256_storeu_si256(compressed + 5, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 22) , 6)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 23) , 15)); - tmp = _mm256_lddqu_si256 (in + 24); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 6, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 25) , 1)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 26) , 10)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 27) , 19)); - tmp = _mm256_lddqu_si256 (in + 28); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 7, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 29) , 5)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 30) , 14)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 31) , 23)); - _mm256_storeu_si256(compressed + 8, w0); -} - - -/* we are going to pack 256 10-bit values, touching 10 256-bit words, using 160 bytes */ -static void avxpackblock10(const uint32_t * pin, __m256i * compressed) { - const __m256i * in = (const __m256i *) pin; - /* we are going to touch 10 256-bit words */ - __m256i w0, w1; - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_lddqu_si256 (in + 0); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 1) , 10)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 2) , 20)); - tmp = _mm256_lddqu_si256 (in + 3); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 30)); - w1 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 4) , 8)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 5) , 18)); - tmp = _mm256_lddqu_si256 (in + 6); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 7) , 6)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 8) , 16)); - tmp = _mm256_lddqu_si256 (in + 9); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 26)); - w1 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 10) , 4)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 11) , 14)); - tmp = _mm256_lddqu_si256 (in + 12); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 3, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 13) , 2)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 14) , 12)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 15) , 22)); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_lddqu_si256 (in + 16); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 17) , 10)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 18) , 20)); - tmp = _mm256_lddqu_si256 (in + 19); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 30)); - w0 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 5, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 20) , 8)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 21) , 18)); - tmp = _mm256_lddqu_si256 (in + 22); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 6, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 23) , 6)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 24) , 16)); - tmp = _mm256_lddqu_si256 (in + 25); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 26)); - w0 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 7, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 26) , 4)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 27) , 14)); - tmp = _mm256_lddqu_si256 (in + 28); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 8, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 29) , 2)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 30) , 12)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 31) , 22)); - _mm256_storeu_si256(compressed + 9, w1); -} - - -/* we are going to pack 256 11-bit values, touching 11 256-bit words, using 176 bytes */ -static void avxpackblock11(const uint32_t * pin, __m256i * compressed) { - const __m256i * in = (const __m256i *) pin; - /* we are going to touch 11 256-bit words */ - __m256i w0, w1; - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_lddqu_si256 (in + 0); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 1) , 11)); - tmp = _mm256_lddqu_si256 (in + 2); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 22)); - w1 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 3) , 1)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 4) , 12)); - tmp = _mm256_lddqu_si256 (in + 5); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 23)); - w0 = _mm256_srli_epi32(tmp,9); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 6) , 2)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 7) , 13)); - tmp = _mm256_lddqu_si256 (in + 8); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 9) , 3)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 10) , 14)); - tmp = _mm256_lddqu_si256 (in + 11); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 25)); - w0 = _mm256_srli_epi32(tmp,7); - _mm256_storeu_si256(compressed + 3, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 12) , 4)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 13) , 15)); - tmp = _mm256_lddqu_si256 (in + 14); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 26)); - w1 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 15) , 5)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 16) , 16)); - tmp = _mm256_lddqu_si256 (in + 17); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 27)); - w0 = _mm256_srli_epi32(tmp,5); - _mm256_storeu_si256(compressed + 5, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 18) , 6)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 19) , 17)); - tmp = _mm256_lddqu_si256 (in + 20); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 6, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 21) , 7)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 22) , 18)); - tmp = _mm256_lddqu_si256 (in + 23); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 29)); - w0 = _mm256_srli_epi32(tmp,3); - _mm256_storeu_si256(compressed + 7, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 24) , 8)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 25) , 19)); - tmp = _mm256_lddqu_si256 (in + 26); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 30)); - w1 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 8, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 27) , 9)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 28) , 20)); - tmp = _mm256_lddqu_si256 (in + 29); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 31)); - w0 = _mm256_srli_epi32(tmp,1); - _mm256_storeu_si256(compressed + 9, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 30) , 10)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 31) , 21)); - _mm256_storeu_si256(compressed + 10, w0); -} - - -/* we are going to pack 256 12-bit values, touching 12 256-bit words, using 192 bytes */ -static void avxpackblock12(const uint32_t * pin, __m256i * compressed) { - const __m256i * in = (const __m256i *) pin; - /* we are going to touch 12 256-bit words */ - __m256i w0, w1; - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_lddqu_si256 (in + 0); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 1) , 12)); - tmp = _mm256_lddqu_si256 (in + 2); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 3) , 4)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 4) , 16)); - tmp = _mm256_lddqu_si256 (in + 5); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 6) , 8)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 7) , 20)); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_lddqu_si256 (in + 8); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 9) , 12)); - tmp = _mm256_lddqu_si256 (in + 10); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 3, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 11) , 4)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 12) , 16)); - tmp = _mm256_lddqu_si256 (in + 13); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 14) , 8)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 15) , 20)); - _mm256_storeu_si256(compressed + 5, w1); - w0 = _mm256_lddqu_si256 (in + 16); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 17) , 12)); - tmp = _mm256_lddqu_si256 (in + 18); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 6, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 19) , 4)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 20) , 16)); - tmp = _mm256_lddqu_si256 (in + 21); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 7, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 22) , 8)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 23) , 20)); - _mm256_storeu_si256(compressed + 8, w0); - w1 = _mm256_lddqu_si256 (in + 24); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 25) , 12)); - tmp = _mm256_lddqu_si256 (in + 26); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 9, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 27) , 4)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 28) , 16)); - tmp = _mm256_lddqu_si256 (in + 29); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 10, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 30) , 8)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 31) , 20)); - _mm256_storeu_si256(compressed + 11, w1); -} - - -/* we are going to pack 256 13-bit values, touching 13 256-bit words, using 208 bytes */ -static void avxpackblock13(const uint32_t * pin, __m256i * compressed) { - const __m256i * in = (const __m256i *) pin; - /* we are going to touch 13 256-bit words */ - __m256i w0, w1; - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_lddqu_si256 (in + 0); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 1) , 13)); - tmp = _mm256_lddqu_si256 (in + 2); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 26)); - w1 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 3) , 7)); - tmp = _mm256_lddqu_si256 (in + 4); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 20)); - w0 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 5) , 1)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 6) , 14)); - tmp = _mm256_lddqu_si256 (in + 7); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 27)); - w1 = _mm256_srli_epi32(tmp,5); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 8) , 8)); - tmp = _mm256_lddqu_si256 (in + 9); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 21)); - w0 = _mm256_srli_epi32(tmp,11); - _mm256_storeu_si256(compressed + 3, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 10) , 2)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 11) , 15)); - tmp = _mm256_lddqu_si256 (in + 12); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 13) , 9)); - tmp = _mm256_lddqu_si256 (in + 14); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 22)); - w0 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 5, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 15) , 3)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 16) , 16)); - tmp = _mm256_lddqu_si256 (in + 17); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 29)); - w1 = _mm256_srli_epi32(tmp,3); - _mm256_storeu_si256(compressed + 6, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 18) , 10)); - tmp = _mm256_lddqu_si256 (in + 19); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 23)); - w0 = _mm256_srli_epi32(tmp,9); - _mm256_storeu_si256(compressed + 7, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 20) , 4)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 21) , 17)); - tmp = _mm256_lddqu_si256 (in + 22); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 30)); - w1 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 8, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 23) , 11)); - tmp = _mm256_lddqu_si256 (in + 24); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 9, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 25) , 5)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 26) , 18)); - tmp = _mm256_lddqu_si256 (in + 27); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 31)); - w1 = _mm256_srli_epi32(tmp,1); - _mm256_storeu_si256(compressed + 10, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 28) , 12)); - tmp = _mm256_lddqu_si256 (in + 29); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 25)); - w0 = _mm256_srli_epi32(tmp,7); - _mm256_storeu_si256(compressed + 11, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 30) , 6)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 31) , 19)); - _mm256_storeu_si256(compressed + 12, w0); -} - - -/* we are going to pack 256 14-bit values, touching 14 256-bit words, using 224 bytes */ -static void avxpackblock14(const uint32_t * pin, __m256i * compressed) { - const __m256i * in = (const __m256i *) pin; - /* we are going to touch 14 256-bit words */ - __m256i w0, w1; - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_lddqu_si256 (in + 0); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 1) , 14)); - tmp = _mm256_lddqu_si256 (in + 2); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 3) , 10)); - tmp = _mm256_lddqu_si256 (in + 4); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 5) , 6)); - tmp = _mm256_lddqu_si256 (in + 6); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 20)); - w1 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 7) , 2)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 8) , 16)); - tmp = _mm256_lddqu_si256 (in + 9); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 30)); - w0 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 3, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 10) , 12)); - tmp = _mm256_lddqu_si256 (in + 11); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 26)); - w1 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 12) , 8)); - tmp = _mm256_lddqu_si256 (in + 13); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 22)); - w0 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 5, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 14) , 4)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 15) , 18)); - _mm256_storeu_si256(compressed + 6, w0); - w1 = _mm256_lddqu_si256 (in + 16); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 17) , 14)); - tmp = _mm256_lddqu_si256 (in + 18); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 7, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 19) , 10)); - tmp = _mm256_lddqu_si256 (in + 20); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 8, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 21) , 6)); - tmp = _mm256_lddqu_si256 (in + 22); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 20)); - w0 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 9, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 23) , 2)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 24) , 16)); - tmp = _mm256_lddqu_si256 (in + 25); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 30)); - w1 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 10, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 26) , 12)); - tmp = _mm256_lddqu_si256 (in + 27); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 26)); - w0 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 11, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 28) , 8)); - tmp = _mm256_lddqu_si256 (in + 29); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 22)); - w1 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 12, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 30) , 4)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 31) , 18)); - _mm256_storeu_si256(compressed + 13, w1); -} - - -/* we are going to pack 256 15-bit values, touching 15 256-bit words, using 240 bytes */ -static void avxpackblock15(const uint32_t * pin, __m256i * compressed) { - const __m256i * in = (const __m256i *) pin; - /* we are going to touch 15 256-bit words */ - __m256i w0, w1; - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_lddqu_si256 (in + 0); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 1) , 15)); - tmp = _mm256_lddqu_si256 (in + 2); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 30)); - w1 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 3) , 13)); - tmp = _mm256_lddqu_si256 (in + 4); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 5) , 11)); - tmp = _mm256_lddqu_si256 (in + 6); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 26)); - w1 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 7) , 9)); - tmp = _mm256_lddqu_si256 (in + 8); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 3, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 9) , 7)); - tmp = _mm256_lddqu_si256 (in + 10); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 22)); - w1 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 11) , 5)); - tmp = _mm256_lddqu_si256 (in + 12); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 20)); - w0 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 5, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 13) , 3)); - tmp = _mm256_lddqu_si256 (in + 14); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 18)); - w1 = _mm256_srli_epi32(tmp,14); - _mm256_storeu_si256(compressed + 6, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 15) , 1)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 16) , 16)); - tmp = _mm256_lddqu_si256 (in + 17); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 31)); - w0 = _mm256_srli_epi32(tmp,1); - _mm256_storeu_si256(compressed + 7, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 18) , 14)); - tmp = _mm256_lddqu_si256 (in + 19); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 29)); - w1 = _mm256_srli_epi32(tmp,3); - _mm256_storeu_si256(compressed + 8, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 20) , 12)); - tmp = _mm256_lddqu_si256 (in + 21); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 27)); - w0 = _mm256_srli_epi32(tmp,5); - _mm256_storeu_si256(compressed + 9, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 22) , 10)); - tmp = _mm256_lddqu_si256 (in + 23); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 25)); - w1 = _mm256_srli_epi32(tmp,7); - _mm256_storeu_si256(compressed + 10, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 24) , 8)); - tmp = _mm256_lddqu_si256 (in + 25); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 23)); - w0 = _mm256_srli_epi32(tmp,9); - _mm256_storeu_si256(compressed + 11, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 26) , 6)); - tmp = _mm256_lddqu_si256 (in + 27); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 21)); - w1 = _mm256_srli_epi32(tmp,11); - _mm256_storeu_si256(compressed + 12, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 28) , 4)); - tmp = _mm256_lddqu_si256 (in + 29); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 19)); - w0 = _mm256_srli_epi32(tmp,13); - _mm256_storeu_si256(compressed + 13, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 30) , 2)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 31) , 17)); - _mm256_storeu_si256(compressed + 14, w0); -} - - -/* we are going to pack 256 16-bit values, touching 16 256-bit words, using 256 bytes */ -static void avxpackblock16(const uint32_t * pin, __m256i * compressed) { - const __m256i * in = (const __m256i *) pin; - /* we are going to touch 16 256-bit words */ - __m256i w0, w1; - w0 = _mm256_lddqu_si256 (in + 0); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 1) , 16)); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_lddqu_si256 (in + 2); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 3) , 16)); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_lddqu_si256 (in + 4); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 5) , 16)); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_lddqu_si256 (in + 6); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 7) , 16)); - _mm256_storeu_si256(compressed + 3, w1); - w0 = _mm256_lddqu_si256 (in + 8); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 9) , 16)); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_lddqu_si256 (in + 10); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 11) , 16)); - _mm256_storeu_si256(compressed + 5, w1); - w0 = _mm256_lddqu_si256 (in + 12); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 13) , 16)); - _mm256_storeu_si256(compressed + 6, w0); - w1 = _mm256_lddqu_si256 (in + 14); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 15) , 16)); - _mm256_storeu_si256(compressed + 7, w1); - w0 = _mm256_lddqu_si256 (in + 16); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 17) , 16)); - _mm256_storeu_si256(compressed + 8, w0); - w1 = _mm256_lddqu_si256 (in + 18); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 19) , 16)); - _mm256_storeu_si256(compressed + 9, w1); - w0 = _mm256_lddqu_si256 (in + 20); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 21) , 16)); - _mm256_storeu_si256(compressed + 10, w0); - w1 = _mm256_lddqu_si256 (in + 22); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 23) , 16)); - _mm256_storeu_si256(compressed + 11, w1); - w0 = _mm256_lddqu_si256 (in + 24); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 25) , 16)); - _mm256_storeu_si256(compressed + 12, w0); - w1 = _mm256_lddqu_si256 (in + 26); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 27) , 16)); - _mm256_storeu_si256(compressed + 13, w1); - w0 = _mm256_lddqu_si256 (in + 28); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 29) , 16)); - _mm256_storeu_si256(compressed + 14, w0); - w1 = _mm256_lddqu_si256 (in + 30); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 31) , 16)); - _mm256_storeu_si256(compressed + 15, w1); -} - - -/* we are going to pack 256 17-bit values, touching 17 256-bit words, using 272 bytes */ -static void avxpackblock17(const uint32_t * pin, __m256i * compressed) { - const __m256i * in = (const __m256i *) pin; - /* we are going to touch 17 256-bit words */ - __m256i w0, w1; - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_lddqu_si256 (in + 0); - tmp = _mm256_lddqu_si256 (in + 1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 17)); - w1 = _mm256_srli_epi32(tmp,15); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 2) , 2)); - tmp = _mm256_lddqu_si256 (in + 3); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 19)); - w0 = _mm256_srli_epi32(tmp,13); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 4) , 4)); - tmp = _mm256_lddqu_si256 (in + 5); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 21)); - w1 = _mm256_srli_epi32(tmp,11); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 6) , 6)); - tmp = _mm256_lddqu_si256 (in + 7); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 23)); - w0 = _mm256_srli_epi32(tmp,9); - _mm256_storeu_si256(compressed + 3, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 8) , 8)); - tmp = _mm256_lddqu_si256 (in + 9); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 25)); - w1 = _mm256_srli_epi32(tmp,7); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 10) , 10)); - tmp = _mm256_lddqu_si256 (in + 11); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 27)); - w0 = _mm256_srli_epi32(tmp,5); - _mm256_storeu_si256(compressed + 5, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 12) , 12)); - tmp = _mm256_lddqu_si256 (in + 13); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 29)); - w1 = _mm256_srli_epi32(tmp,3); - _mm256_storeu_si256(compressed + 6, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 14) , 14)); - tmp = _mm256_lddqu_si256 (in + 15); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 31)); - w0 = _mm256_srli_epi32(tmp,1); - _mm256_storeu_si256(compressed + 7, w1); - tmp = _mm256_lddqu_si256 (in + 16); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 16)); - w1 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 8, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 17) , 1)); - tmp = _mm256_lddqu_si256 (in + 18); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 18)); - w0 = _mm256_srli_epi32(tmp,14); - _mm256_storeu_si256(compressed + 9, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 19) , 3)); - tmp = _mm256_lddqu_si256 (in + 20); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 20)); - w1 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 10, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 21) , 5)); - tmp = _mm256_lddqu_si256 (in + 22); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 22)); - w0 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 11, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 23) , 7)); - tmp = _mm256_lddqu_si256 (in + 24); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 12, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 25) , 9)); - tmp = _mm256_lddqu_si256 (in + 26); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 26)); - w0 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 13, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 27) , 11)); - tmp = _mm256_lddqu_si256 (in + 28); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 14, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 29) , 13)); - tmp = _mm256_lddqu_si256 (in + 30); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 30)); - w0 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 15, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 31) , 15)); - _mm256_storeu_si256(compressed + 16, w0); -} - - -/* we are going to pack 256 18-bit values, touching 18 256-bit words, using 288 bytes */ -static void avxpackblock18(const uint32_t * pin, __m256i * compressed) { - const __m256i * in = (const __m256i *) pin; - /* we are going to touch 18 256-bit words */ - __m256i w0, w1; - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_lddqu_si256 (in + 0); - tmp = _mm256_lddqu_si256 (in + 1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 18)); - w1 = _mm256_srli_epi32(tmp,14); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 2) , 4)); - tmp = _mm256_lddqu_si256 (in + 3); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 22)); - w0 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 4) , 8)); - tmp = _mm256_lddqu_si256 (in + 5); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 26)); - w1 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 6) , 12)); - tmp = _mm256_lddqu_si256 (in + 7); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 30)); - w0 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 3, w1); - tmp = _mm256_lddqu_si256 (in + 8); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 16)); - w1 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 9) , 2)); - tmp = _mm256_lddqu_si256 (in + 10); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 20)); - w0 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 5, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 11) , 6)); - tmp = _mm256_lddqu_si256 (in + 12); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 6, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 13) , 10)); - tmp = _mm256_lddqu_si256 (in + 14); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 7, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 15) , 14)); - _mm256_storeu_si256(compressed + 8, w0); - w1 = _mm256_lddqu_si256 (in + 16); - tmp = _mm256_lddqu_si256 (in + 17); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 18)); - w0 = _mm256_srli_epi32(tmp,14); - _mm256_storeu_si256(compressed + 9, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 18) , 4)); - tmp = _mm256_lddqu_si256 (in + 19); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 22)); - w1 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 10, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 20) , 8)); - tmp = _mm256_lddqu_si256 (in + 21); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 26)); - w0 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 11, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 22) , 12)); - tmp = _mm256_lddqu_si256 (in + 23); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 30)); - w1 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 12, w0); - tmp = _mm256_lddqu_si256 (in + 24); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 16)); - w0 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 13, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 25) , 2)); - tmp = _mm256_lddqu_si256 (in + 26); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 20)); - w1 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 14, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 27) , 6)); - tmp = _mm256_lddqu_si256 (in + 28); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 15, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 29) , 10)); - tmp = _mm256_lddqu_si256 (in + 30); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 16, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 31) , 14)); - _mm256_storeu_si256(compressed + 17, w1); -} - - -/* we are going to pack 256 19-bit values, touching 19 256-bit words, using 304 bytes */ -static void avxpackblock19(const uint32_t * pin, __m256i * compressed) { - const __m256i * in = (const __m256i *) pin; - /* we are going to touch 19 256-bit words */ - __m256i w0, w1; - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_lddqu_si256 (in + 0); - tmp = _mm256_lddqu_si256 (in + 1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 19)); - w1 = _mm256_srli_epi32(tmp,13); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 2) , 6)); - tmp = _mm256_lddqu_si256 (in + 3); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 25)); - w0 = _mm256_srli_epi32(tmp,7); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 4) , 12)); - tmp = _mm256_lddqu_si256 (in + 5); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 31)); - w1 = _mm256_srli_epi32(tmp,1); - _mm256_storeu_si256(compressed + 2, w0); - tmp = _mm256_lddqu_si256 (in + 6); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 18)); - w0 = _mm256_srli_epi32(tmp,14); - _mm256_storeu_si256(compressed + 3, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 7) , 5)); - tmp = _mm256_lddqu_si256 (in + 8); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 9) , 11)); - tmp = _mm256_lddqu_si256 (in + 10); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 30)); - w0 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 5, w1); - tmp = _mm256_lddqu_si256 (in + 11); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 17)); - w1 = _mm256_srli_epi32(tmp,15); - _mm256_storeu_si256(compressed + 6, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 12) , 4)); - tmp = _mm256_lddqu_si256 (in + 13); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 23)); - w0 = _mm256_srli_epi32(tmp,9); - _mm256_storeu_si256(compressed + 7, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 14) , 10)); - tmp = _mm256_lddqu_si256 (in + 15); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 29)); - w1 = _mm256_srli_epi32(tmp,3); - _mm256_storeu_si256(compressed + 8, w0); - tmp = _mm256_lddqu_si256 (in + 16); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 16)); - w0 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 9, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 17) , 3)); - tmp = _mm256_lddqu_si256 (in + 18); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 22)); - w1 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 10, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 19) , 9)); - tmp = _mm256_lddqu_si256 (in + 20); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 11, w1); - tmp = _mm256_lddqu_si256 (in + 21); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 15)); - w1 = _mm256_srli_epi32(tmp,17); - _mm256_storeu_si256(compressed + 12, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 22) , 2)); - tmp = _mm256_lddqu_si256 (in + 23); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 21)); - w0 = _mm256_srli_epi32(tmp,11); - _mm256_storeu_si256(compressed + 13, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 24) , 8)); - tmp = _mm256_lddqu_si256 (in + 25); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 27)); - w1 = _mm256_srli_epi32(tmp,5); - _mm256_storeu_si256(compressed + 14, w0); - tmp = _mm256_lddqu_si256 (in + 26); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 14)); - w0 = _mm256_srli_epi32(tmp,18); - _mm256_storeu_si256(compressed + 15, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 27) , 1)); - tmp = _mm256_lddqu_si256 (in + 28); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 20)); - w1 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 16, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 29) , 7)); - tmp = _mm256_lddqu_si256 (in + 30); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 26)); - w0 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 17, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 31) , 13)); - _mm256_storeu_si256(compressed + 18, w0); -} - - -/* we are going to pack 256 20-bit values, touching 20 256-bit words, using 320 bytes */ -static void avxpackblock20(const uint32_t * pin, __m256i * compressed) { - const __m256i * in = (const __m256i *) pin; - /* we are going to touch 20 256-bit words */ - __m256i w0, w1; - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_lddqu_si256 (in + 0); - tmp = _mm256_lddqu_si256 (in + 1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 20)); - w1 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 2) , 8)); - tmp = _mm256_lddqu_si256 (in + 3); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 1, w1); - tmp = _mm256_lddqu_si256 (in + 4); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 16)); - w1 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 5) , 4)); - tmp = _mm256_lddqu_si256 (in + 6); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 3, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 7) , 12)); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_lddqu_si256 (in + 8); - tmp = _mm256_lddqu_si256 (in + 9); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 20)); - w0 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 5, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 10) , 8)); - tmp = _mm256_lddqu_si256 (in + 11); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 6, w0); - tmp = _mm256_lddqu_si256 (in + 12); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 16)); - w0 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 7, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 13) , 4)); - tmp = _mm256_lddqu_si256 (in + 14); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 8, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 15) , 12)); - _mm256_storeu_si256(compressed + 9, w1); - w0 = _mm256_lddqu_si256 (in + 16); - tmp = _mm256_lddqu_si256 (in + 17); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 20)); - w1 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 10, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 18) , 8)); - tmp = _mm256_lddqu_si256 (in + 19); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 11, w1); - tmp = _mm256_lddqu_si256 (in + 20); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 16)); - w1 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 12, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 21) , 4)); - tmp = _mm256_lddqu_si256 (in + 22); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 13, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 23) , 12)); - _mm256_storeu_si256(compressed + 14, w0); - w1 = _mm256_lddqu_si256 (in + 24); - tmp = _mm256_lddqu_si256 (in + 25); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 20)); - w0 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 15, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 26) , 8)); - tmp = _mm256_lddqu_si256 (in + 27); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 16, w0); - tmp = _mm256_lddqu_si256 (in + 28); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 16)); - w0 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 17, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 29) , 4)); - tmp = _mm256_lddqu_si256 (in + 30); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 18, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 31) , 12)); - _mm256_storeu_si256(compressed + 19, w1); -} - - -/* we are going to pack 256 21-bit values, touching 21 256-bit words, using 336 bytes */ -static void avxpackblock21(const uint32_t * pin, __m256i * compressed) { - const __m256i * in = (const __m256i *) pin; - /* we are going to touch 21 256-bit words */ - __m256i w0, w1; - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_lddqu_si256 (in + 0); - tmp = _mm256_lddqu_si256 (in + 1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 21)); - w1 = _mm256_srli_epi32(tmp,11); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 2) , 10)); - tmp = _mm256_lddqu_si256 (in + 3); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 31)); - w0 = _mm256_srli_epi32(tmp,1); - _mm256_storeu_si256(compressed + 1, w1); - tmp = _mm256_lddqu_si256 (in + 4); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 20)); - w1 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 5) , 9)); - tmp = _mm256_lddqu_si256 (in + 6); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 30)); - w0 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 3, w1); - tmp = _mm256_lddqu_si256 (in + 7); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 19)); - w1 = _mm256_srli_epi32(tmp,13); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 8) , 8)); - tmp = _mm256_lddqu_si256 (in + 9); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 29)); - w0 = _mm256_srli_epi32(tmp,3); - _mm256_storeu_si256(compressed + 5, w1); - tmp = _mm256_lddqu_si256 (in + 10); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 18)); - w1 = _mm256_srli_epi32(tmp,14); - _mm256_storeu_si256(compressed + 6, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 11) , 7)); - tmp = _mm256_lddqu_si256 (in + 12); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 7, w1); - tmp = _mm256_lddqu_si256 (in + 13); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 17)); - w1 = _mm256_srli_epi32(tmp,15); - _mm256_storeu_si256(compressed + 8, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 14) , 6)); - tmp = _mm256_lddqu_si256 (in + 15); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 27)); - w0 = _mm256_srli_epi32(tmp,5); - _mm256_storeu_si256(compressed + 9, w1); - tmp = _mm256_lddqu_si256 (in + 16); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 16)); - w1 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 10, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 17) , 5)); - tmp = _mm256_lddqu_si256 (in + 18); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 26)); - w0 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 11, w1); - tmp = _mm256_lddqu_si256 (in + 19); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 15)); - w1 = _mm256_srli_epi32(tmp,17); - _mm256_storeu_si256(compressed + 12, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 20) , 4)); - tmp = _mm256_lddqu_si256 (in + 21); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 25)); - w0 = _mm256_srli_epi32(tmp,7); - _mm256_storeu_si256(compressed + 13, w1); - tmp = _mm256_lddqu_si256 (in + 22); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 14)); - w1 = _mm256_srli_epi32(tmp,18); - _mm256_storeu_si256(compressed + 14, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 23) , 3)); - tmp = _mm256_lddqu_si256 (in + 24); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 15, w1); - tmp = _mm256_lddqu_si256 (in + 25); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 13)); - w1 = _mm256_srli_epi32(tmp,19); - _mm256_storeu_si256(compressed + 16, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 26) , 2)); - tmp = _mm256_lddqu_si256 (in + 27); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 23)); - w0 = _mm256_srli_epi32(tmp,9); - _mm256_storeu_si256(compressed + 17, w1); - tmp = _mm256_lddqu_si256 (in + 28); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 12)); - w1 = _mm256_srli_epi32(tmp,20); - _mm256_storeu_si256(compressed + 18, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 29) , 1)); - tmp = _mm256_lddqu_si256 (in + 30); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 22)); - w0 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 19, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 31) , 11)); - _mm256_storeu_si256(compressed + 20, w0); -} - - -/* we are going to pack 256 22-bit values, touching 22 256-bit words, using 352 bytes */ -static void avxpackblock22(const uint32_t * pin, __m256i * compressed) { - const __m256i * in = (const __m256i *) pin; - /* we are going to touch 22 256-bit words */ - __m256i w0, w1; - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_lddqu_si256 (in + 0); - tmp = _mm256_lddqu_si256 (in + 1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 22)); - w1 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 0, w0); - tmp = _mm256_lddqu_si256 (in + 2); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 12)); - w0 = _mm256_srli_epi32(tmp,20); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 3) , 2)); - tmp = _mm256_lddqu_si256 (in + 4); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 2, w0); - tmp = _mm256_lddqu_si256 (in + 5); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 14)); - w0 = _mm256_srli_epi32(tmp,18); - _mm256_storeu_si256(compressed + 3, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 6) , 4)); - tmp = _mm256_lddqu_si256 (in + 7); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 26)); - w1 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 4, w0); - tmp = _mm256_lddqu_si256 (in + 8); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 16)); - w0 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 5, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 9) , 6)); - tmp = _mm256_lddqu_si256 (in + 10); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 6, w0); - tmp = _mm256_lddqu_si256 (in + 11); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 18)); - w0 = _mm256_srli_epi32(tmp,14); - _mm256_storeu_si256(compressed + 7, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 12) , 8)); - tmp = _mm256_lddqu_si256 (in + 13); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 30)); - w1 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 8, w0); - tmp = _mm256_lddqu_si256 (in + 14); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 20)); - w0 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 9, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 15) , 10)); - _mm256_storeu_si256(compressed + 10, w0); - w1 = _mm256_lddqu_si256 (in + 16); - tmp = _mm256_lddqu_si256 (in + 17); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 22)); - w0 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 11, w1); - tmp = _mm256_lddqu_si256 (in + 18); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 12)); - w1 = _mm256_srli_epi32(tmp,20); - _mm256_storeu_si256(compressed + 12, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 19) , 2)); - tmp = _mm256_lddqu_si256 (in + 20); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 13, w1); - tmp = _mm256_lddqu_si256 (in + 21); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 14)); - w1 = _mm256_srli_epi32(tmp,18); - _mm256_storeu_si256(compressed + 14, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 22) , 4)); - tmp = _mm256_lddqu_si256 (in + 23); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 26)); - w0 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 15, w1); - tmp = _mm256_lddqu_si256 (in + 24); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 16)); - w1 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 16, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 25) , 6)); - tmp = _mm256_lddqu_si256 (in + 26); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 17, w1); - tmp = _mm256_lddqu_si256 (in + 27); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 18)); - w1 = _mm256_srli_epi32(tmp,14); - _mm256_storeu_si256(compressed + 18, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 28) , 8)); - tmp = _mm256_lddqu_si256 (in + 29); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 30)); - w0 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 19, w1); - tmp = _mm256_lddqu_si256 (in + 30); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 20)); - w1 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 20, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 31) , 10)); - _mm256_storeu_si256(compressed + 21, w1); -} - - -/* we are going to pack 256 23-bit values, touching 23 256-bit words, using 368 bytes */ -static void avxpackblock23(const uint32_t * pin, __m256i * compressed) { - const __m256i * in = (const __m256i *) pin; - /* we are going to touch 23 256-bit words */ - __m256i w0, w1; - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_lddqu_si256 (in + 0); - tmp = _mm256_lddqu_si256 (in + 1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 23)); - w1 = _mm256_srli_epi32(tmp,9); - _mm256_storeu_si256(compressed + 0, w0); - tmp = _mm256_lddqu_si256 (in + 2); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 14)); - w0 = _mm256_srli_epi32(tmp,18); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 3) , 5)); - tmp = _mm256_lddqu_si256 (in + 4); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 2, w0); - tmp = _mm256_lddqu_si256 (in + 5); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 19)); - w0 = _mm256_srli_epi32(tmp,13); - _mm256_storeu_si256(compressed + 3, w1); - tmp = _mm256_lddqu_si256 (in + 6); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 10)); - w1 = _mm256_srli_epi32(tmp,22); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 7) , 1)); - tmp = _mm256_lddqu_si256 (in + 8); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 5, w1); - tmp = _mm256_lddqu_si256 (in + 9); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 15)); - w1 = _mm256_srli_epi32(tmp,17); - _mm256_storeu_si256(compressed + 6, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 10) , 6)); - tmp = _mm256_lddqu_si256 (in + 11); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 29)); - w0 = _mm256_srli_epi32(tmp,3); - _mm256_storeu_si256(compressed + 7, w1); - tmp = _mm256_lddqu_si256 (in + 12); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 20)); - w1 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 8, w0); - tmp = _mm256_lddqu_si256 (in + 13); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 11)); - w0 = _mm256_srli_epi32(tmp,21); - _mm256_storeu_si256(compressed + 9, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 14) , 2)); - tmp = _mm256_lddqu_si256 (in + 15); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 25)); - w1 = _mm256_srli_epi32(tmp,7); - _mm256_storeu_si256(compressed + 10, w0); - tmp = _mm256_lddqu_si256 (in + 16); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 16)); - w0 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 11, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 17) , 7)); - tmp = _mm256_lddqu_si256 (in + 18); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 30)); - w1 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 12, w0); - tmp = _mm256_lddqu_si256 (in + 19); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 21)); - w0 = _mm256_srli_epi32(tmp,11); - _mm256_storeu_si256(compressed + 13, w1); - tmp = _mm256_lddqu_si256 (in + 20); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 12)); - w1 = _mm256_srli_epi32(tmp,20); - _mm256_storeu_si256(compressed + 14, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 21) , 3)); - tmp = _mm256_lddqu_si256 (in + 22); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 26)); - w0 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 15, w1); - tmp = _mm256_lddqu_si256 (in + 23); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 17)); - w1 = _mm256_srli_epi32(tmp,15); - _mm256_storeu_si256(compressed + 16, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 24) , 8)); - tmp = _mm256_lddqu_si256 (in + 25); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 31)); - w0 = _mm256_srli_epi32(tmp,1); - _mm256_storeu_si256(compressed + 17, w1); - tmp = _mm256_lddqu_si256 (in + 26); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 22)); - w1 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 18, w0); - tmp = _mm256_lddqu_si256 (in + 27); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 13)); - w0 = _mm256_srli_epi32(tmp,19); - _mm256_storeu_si256(compressed + 19, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 28) , 4)); - tmp = _mm256_lddqu_si256 (in + 29); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 27)); - w1 = _mm256_srli_epi32(tmp,5); - _mm256_storeu_si256(compressed + 20, w0); - tmp = _mm256_lddqu_si256 (in + 30); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 18)); - w0 = _mm256_srli_epi32(tmp,14); - _mm256_storeu_si256(compressed + 21, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 31) , 9)); - _mm256_storeu_si256(compressed + 22, w0); -} - - -/* we are going to pack 256 24-bit values, touching 24 256-bit words, using 384 bytes */ -static void avxpackblock24(const uint32_t * pin, __m256i * compressed) { - const __m256i * in = (const __m256i *) pin; - /* we are going to touch 24 256-bit words */ - __m256i w0, w1; - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_lddqu_si256 (in + 0); - tmp = _mm256_lddqu_si256 (in + 1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 0, w0); - tmp = _mm256_lddqu_si256 (in + 2); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 16)); - w0 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 3) , 8)); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_lddqu_si256 (in + 4); - tmp = _mm256_lddqu_si256 (in + 5); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 3, w1); - tmp = _mm256_lddqu_si256 (in + 6); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 16)); - w1 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 7) , 8)); - _mm256_storeu_si256(compressed + 5, w1); - w0 = _mm256_lddqu_si256 (in + 8); - tmp = _mm256_lddqu_si256 (in + 9); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 6, w0); - tmp = _mm256_lddqu_si256 (in + 10); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 16)); - w0 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 7, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 11) , 8)); - _mm256_storeu_si256(compressed + 8, w0); - w1 = _mm256_lddqu_si256 (in + 12); - tmp = _mm256_lddqu_si256 (in + 13); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 9, w1); - tmp = _mm256_lddqu_si256 (in + 14); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 16)); - w1 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 10, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 15) , 8)); - _mm256_storeu_si256(compressed + 11, w1); - w0 = _mm256_lddqu_si256 (in + 16); - tmp = _mm256_lddqu_si256 (in + 17); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 12, w0); - tmp = _mm256_lddqu_si256 (in + 18); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 16)); - w0 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 13, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 19) , 8)); - _mm256_storeu_si256(compressed + 14, w0); - w1 = _mm256_lddqu_si256 (in + 20); - tmp = _mm256_lddqu_si256 (in + 21); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 15, w1); - tmp = _mm256_lddqu_si256 (in + 22); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 16)); - w1 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 16, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 23) , 8)); - _mm256_storeu_si256(compressed + 17, w1); - w0 = _mm256_lddqu_si256 (in + 24); - tmp = _mm256_lddqu_si256 (in + 25); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 18, w0); - tmp = _mm256_lddqu_si256 (in + 26); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 16)); - w0 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 19, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 27) , 8)); - _mm256_storeu_si256(compressed + 20, w0); - w1 = _mm256_lddqu_si256 (in + 28); - tmp = _mm256_lddqu_si256 (in + 29); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 21, w1); - tmp = _mm256_lddqu_si256 (in + 30); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 16)); - w1 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 22, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 31) , 8)); - _mm256_storeu_si256(compressed + 23, w1); -} - - -/* we are going to pack 256 25-bit values, touching 25 256-bit words, using 400 bytes */ -static void avxpackblock25(const uint32_t * pin, __m256i * compressed) { - const __m256i * in = (const __m256i *) pin; - /* we are going to touch 25 256-bit words */ - __m256i w0, w1; - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_lddqu_si256 (in + 0); - tmp = _mm256_lddqu_si256 (in + 1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 25)); - w1 = _mm256_srli_epi32(tmp,7); - _mm256_storeu_si256(compressed + 0, w0); - tmp = _mm256_lddqu_si256 (in + 2); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 18)); - w0 = _mm256_srli_epi32(tmp,14); - _mm256_storeu_si256(compressed + 1, w1); - tmp = _mm256_lddqu_si256 (in + 3); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 11)); - w1 = _mm256_srli_epi32(tmp,21); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 4) , 4)); - tmp = _mm256_lddqu_si256 (in + 5); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 29)); - w0 = _mm256_srli_epi32(tmp,3); - _mm256_storeu_si256(compressed + 3, w1); - tmp = _mm256_lddqu_si256 (in + 6); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 22)); - w1 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 4, w0); - tmp = _mm256_lddqu_si256 (in + 7); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 15)); - w0 = _mm256_srli_epi32(tmp,17); - _mm256_storeu_si256(compressed + 5, w1); - tmp = _mm256_lddqu_si256 (in + 8); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 8)); - w1 = _mm256_srli_epi32(tmp,24); - _mm256_storeu_si256(compressed + 6, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 9) , 1)); - tmp = _mm256_lddqu_si256 (in + 10); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 26)); - w0 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 7, w1); - tmp = _mm256_lddqu_si256 (in + 11); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 19)); - w1 = _mm256_srli_epi32(tmp,13); - _mm256_storeu_si256(compressed + 8, w0); - tmp = _mm256_lddqu_si256 (in + 12); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 12)); - w0 = _mm256_srli_epi32(tmp,20); - _mm256_storeu_si256(compressed + 9, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 13) , 5)); - tmp = _mm256_lddqu_si256 (in + 14); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 30)); - w1 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 10, w0); - tmp = _mm256_lddqu_si256 (in + 15); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 23)); - w0 = _mm256_srli_epi32(tmp,9); - _mm256_storeu_si256(compressed + 11, w1); - tmp = _mm256_lddqu_si256 (in + 16); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 16)); - w1 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 12, w0); - tmp = _mm256_lddqu_si256 (in + 17); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 9)); - w0 = _mm256_srli_epi32(tmp,23); - _mm256_storeu_si256(compressed + 13, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 18) , 2)); - tmp = _mm256_lddqu_si256 (in + 19); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 27)); - w1 = _mm256_srli_epi32(tmp,5); - _mm256_storeu_si256(compressed + 14, w0); - tmp = _mm256_lddqu_si256 (in + 20); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 20)); - w0 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 15, w1); - tmp = _mm256_lddqu_si256 (in + 21); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 13)); - w1 = _mm256_srli_epi32(tmp,19); - _mm256_storeu_si256(compressed + 16, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 22) , 6)); - tmp = _mm256_lddqu_si256 (in + 23); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 31)); - w0 = _mm256_srli_epi32(tmp,1); - _mm256_storeu_si256(compressed + 17, w1); - tmp = _mm256_lddqu_si256 (in + 24); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 18, w0); - tmp = _mm256_lddqu_si256 (in + 25); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 17)); - w0 = _mm256_srli_epi32(tmp,15); - _mm256_storeu_si256(compressed + 19, w1); - tmp = _mm256_lddqu_si256 (in + 26); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 10)); - w1 = _mm256_srli_epi32(tmp,22); - _mm256_storeu_si256(compressed + 20, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 27) , 3)); - tmp = _mm256_lddqu_si256 (in + 28); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 21, w1); - tmp = _mm256_lddqu_si256 (in + 29); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 21)); - w1 = _mm256_srli_epi32(tmp,11); - _mm256_storeu_si256(compressed + 22, w0); - tmp = _mm256_lddqu_si256 (in + 30); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 14)); - w0 = _mm256_srli_epi32(tmp,18); - _mm256_storeu_si256(compressed + 23, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 31) , 7)); - _mm256_storeu_si256(compressed + 24, w0); -} - - -/* we are going to pack 256 26-bit values, touching 26 256-bit words, using 416 bytes */ -static void avxpackblock26(const uint32_t * pin, __m256i * compressed) { - const __m256i * in = (const __m256i *) pin; - /* we are going to touch 26 256-bit words */ - __m256i w0, w1; - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_lddqu_si256 (in + 0); - tmp = _mm256_lddqu_si256 (in + 1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 26)); - w1 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 0, w0); - tmp = _mm256_lddqu_si256 (in + 2); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 20)); - w0 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 1, w1); - tmp = _mm256_lddqu_si256 (in + 3); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 14)); - w1 = _mm256_srli_epi32(tmp,18); - _mm256_storeu_si256(compressed + 2, w0); - tmp = _mm256_lddqu_si256 (in + 4); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 8)); - w0 = _mm256_srli_epi32(tmp,24); - _mm256_storeu_si256(compressed + 3, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 5) , 2)); - tmp = _mm256_lddqu_si256 (in + 6); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 4, w0); - tmp = _mm256_lddqu_si256 (in + 7); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 22)); - w0 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 5, w1); - tmp = _mm256_lddqu_si256 (in + 8); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 16)); - w1 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 6, w0); - tmp = _mm256_lddqu_si256 (in + 9); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 10)); - w0 = _mm256_srli_epi32(tmp,22); - _mm256_storeu_si256(compressed + 7, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 10) , 4)); - tmp = _mm256_lddqu_si256 (in + 11); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 30)); - w1 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 8, w0); - tmp = _mm256_lddqu_si256 (in + 12); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 9, w1); - tmp = _mm256_lddqu_si256 (in + 13); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 18)); - w1 = _mm256_srli_epi32(tmp,14); - _mm256_storeu_si256(compressed + 10, w0); - tmp = _mm256_lddqu_si256 (in + 14); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 12)); - w0 = _mm256_srli_epi32(tmp,20); - _mm256_storeu_si256(compressed + 11, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 15) , 6)); - _mm256_storeu_si256(compressed + 12, w0); - w1 = _mm256_lddqu_si256 (in + 16); - tmp = _mm256_lddqu_si256 (in + 17); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 26)); - w0 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 13, w1); - tmp = _mm256_lddqu_si256 (in + 18); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 20)); - w1 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 14, w0); - tmp = _mm256_lddqu_si256 (in + 19); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 14)); - w0 = _mm256_srli_epi32(tmp,18); - _mm256_storeu_si256(compressed + 15, w1); - tmp = _mm256_lddqu_si256 (in + 20); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 8)); - w1 = _mm256_srli_epi32(tmp,24); - _mm256_storeu_si256(compressed + 16, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 21) , 2)); - tmp = _mm256_lddqu_si256 (in + 22); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 17, w1); - tmp = _mm256_lddqu_si256 (in + 23); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 22)); - w1 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 18, w0); - tmp = _mm256_lddqu_si256 (in + 24); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 16)); - w0 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 19, w1); - tmp = _mm256_lddqu_si256 (in + 25); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 10)); - w1 = _mm256_srli_epi32(tmp,22); - _mm256_storeu_si256(compressed + 20, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 26) , 4)); - tmp = _mm256_lddqu_si256 (in + 27); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 30)); - w0 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 21, w1); - tmp = _mm256_lddqu_si256 (in + 28); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 22, w0); - tmp = _mm256_lddqu_si256 (in + 29); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 18)); - w0 = _mm256_srli_epi32(tmp,14); - _mm256_storeu_si256(compressed + 23, w1); - tmp = _mm256_lddqu_si256 (in + 30); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 12)); - w1 = _mm256_srli_epi32(tmp,20); - _mm256_storeu_si256(compressed + 24, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 31) , 6)); - _mm256_storeu_si256(compressed + 25, w1); -} - - -/* we are going to pack 256 27-bit values, touching 27 256-bit words, using 432 bytes */ -static void avxpackblock27(const uint32_t * pin, __m256i * compressed) { - const __m256i * in = (const __m256i *) pin; - /* we are going to touch 27 256-bit words */ - __m256i w0, w1; - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_lddqu_si256 (in + 0); - tmp = _mm256_lddqu_si256 (in + 1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 27)); - w1 = _mm256_srli_epi32(tmp,5); - _mm256_storeu_si256(compressed + 0, w0); - tmp = _mm256_lddqu_si256 (in + 2); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 22)); - w0 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 1, w1); - tmp = _mm256_lddqu_si256 (in + 3); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 17)); - w1 = _mm256_srli_epi32(tmp,15); - _mm256_storeu_si256(compressed + 2, w0); - tmp = _mm256_lddqu_si256 (in + 4); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 12)); - w0 = _mm256_srli_epi32(tmp,20); - _mm256_storeu_si256(compressed + 3, w1); - tmp = _mm256_lddqu_si256 (in + 5); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 7)); - w1 = _mm256_srli_epi32(tmp,25); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 6) , 2)); - tmp = _mm256_lddqu_si256 (in + 7); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 29)); - w0 = _mm256_srli_epi32(tmp,3); - _mm256_storeu_si256(compressed + 5, w1); - tmp = _mm256_lddqu_si256 (in + 8); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 6, w0); - tmp = _mm256_lddqu_si256 (in + 9); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 19)); - w0 = _mm256_srli_epi32(tmp,13); - _mm256_storeu_si256(compressed + 7, w1); - tmp = _mm256_lddqu_si256 (in + 10); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 14)); - w1 = _mm256_srli_epi32(tmp,18); - _mm256_storeu_si256(compressed + 8, w0); - tmp = _mm256_lddqu_si256 (in + 11); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 9)); - w0 = _mm256_srli_epi32(tmp,23); - _mm256_storeu_si256(compressed + 9, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 12) , 4)); - tmp = _mm256_lddqu_si256 (in + 13); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 31)); - w1 = _mm256_srli_epi32(tmp,1); - _mm256_storeu_si256(compressed + 10, w0); - tmp = _mm256_lddqu_si256 (in + 14); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 26)); - w0 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 11, w1); - tmp = _mm256_lddqu_si256 (in + 15); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 21)); - w1 = _mm256_srli_epi32(tmp,11); - _mm256_storeu_si256(compressed + 12, w0); - tmp = _mm256_lddqu_si256 (in + 16); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 16)); - w0 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 13, w1); - tmp = _mm256_lddqu_si256 (in + 17); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 11)); - w1 = _mm256_srli_epi32(tmp,21); - _mm256_storeu_si256(compressed + 14, w0); - tmp = _mm256_lddqu_si256 (in + 18); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 6)); - w0 = _mm256_srli_epi32(tmp,26); - _mm256_storeu_si256(compressed + 15, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 19) , 1)); - tmp = _mm256_lddqu_si256 (in + 20); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 16, w0); - tmp = _mm256_lddqu_si256 (in + 21); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 23)); - w0 = _mm256_srli_epi32(tmp,9); - _mm256_storeu_si256(compressed + 17, w1); - tmp = _mm256_lddqu_si256 (in + 22); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 18)); - w1 = _mm256_srli_epi32(tmp,14); - _mm256_storeu_si256(compressed + 18, w0); - tmp = _mm256_lddqu_si256 (in + 23); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 13)); - w0 = _mm256_srli_epi32(tmp,19); - _mm256_storeu_si256(compressed + 19, w1); - tmp = _mm256_lddqu_si256 (in + 24); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 8)); - w1 = _mm256_srli_epi32(tmp,24); - _mm256_storeu_si256(compressed + 20, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 25) , 3)); - tmp = _mm256_lddqu_si256 (in + 26); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 30)); - w0 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 21, w1); - tmp = _mm256_lddqu_si256 (in + 27); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 25)); - w1 = _mm256_srli_epi32(tmp,7); - _mm256_storeu_si256(compressed + 22, w0); - tmp = _mm256_lddqu_si256 (in + 28); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 20)); - w0 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 23, w1); - tmp = _mm256_lddqu_si256 (in + 29); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 15)); - w1 = _mm256_srli_epi32(tmp,17); - _mm256_storeu_si256(compressed + 24, w0); - tmp = _mm256_lddqu_si256 (in + 30); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 10)); - w0 = _mm256_srli_epi32(tmp,22); - _mm256_storeu_si256(compressed + 25, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 31) , 5)); - _mm256_storeu_si256(compressed + 26, w0); -} - - -/* we are going to pack 256 28-bit values, touching 28 256-bit words, using 448 bytes */ -static void avxpackblock28(const uint32_t * pin, __m256i * compressed) { - const __m256i * in = (const __m256i *) pin; - /* we are going to touch 28 256-bit words */ - __m256i w0, w1; - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_lddqu_si256 (in + 0); - tmp = _mm256_lddqu_si256 (in + 1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 0, w0); - tmp = _mm256_lddqu_si256 (in + 2); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 1, w1); - tmp = _mm256_lddqu_si256 (in + 3); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 20)); - w1 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 2, w0); - tmp = _mm256_lddqu_si256 (in + 4); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 16)); - w0 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 3, w1); - tmp = _mm256_lddqu_si256 (in + 5); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 12)); - w1 = _mm256_srli_epi32(tmp,20); - _mm256_storeu_si256(compressed + 4, w0); - tmp = _mm256_lddqu_si256 (in + 6); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 8)); - w0 = _mm256_srli_epi32(tmp,24); - _mm256_storeu_si256(compressed + 5, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 7) , 4)); - _mm256_storeu_si256(compressed + 6, w0); - w1 = _mm256_lddqu_si256 (in + 8); - tmp = _mm256_lddqu_si256 (in + 9); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 7, w1); - tmp = _mm256_lddqu_si256 (in + 10); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 8, w0); - tmp = _mm256_lddqu_si256 (in + 11); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 20)); - w0 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 9, w1); - tmp = _mm256_lddqu_si256 (in + 12); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 16)); - w1 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 10, w0); - tmp = _mm256_lddqu_si256 (in + 13); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 12)); - w0 = _mm256_srli_epi32(tmp,20); - _mm256_storeu_si256(compressed + 11, w1); - tmp = _mm256_lddqu_si256 (in + 14); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 8)); - w1 = _mm256_srli_epi32(tmp,24); - _mm256_storeu_si256(compressed + 12, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 15) , 4)); - _mm256_storeu_si256(compressed + 13, w1); - w0 = _mm256_lddqu_si256 (in + 16); - tmp = _mm256_lddqu_si256 (in + 17); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 14, w0); - tmp = _mm256_lddqu_si256 (in + 18); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 15, w1); - tmp = _mm256_lddqu_si256 (in + 19); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 20)); - w1 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 16, w0); - tmp = _mm256_lddqu_si256 (in + 20); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 16)); - w0 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 17, w1); - tmp = _mm256_lddqu_si256 (in + 21); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 12)); - w1 = _mm256_srli_epi32(tmp,20); - _mm256_storeu_si256(compressed + 18, w0); - tmp = _mm256_lddqu_si256 (in + 22); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 8)); - w0 = _mm256_srli_epi32(tmp,24); - _mm256_storeu_si256(compressed + 19, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 23) , 4)); - _mm256_storeu_si256(compressed + 20, w0); - w1 = _mm256_lddqu_si256 (in + 24); - tmp = _mm256_lddqu_si256 (in + 25); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 21, w1); - tmp = _mm256_lddqu_si256 (in + 26); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 22, w0); - tmp = _mm256_lddqu_si256 (in + 27); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 20)); - w0 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 23, w1); - tmp = _mm256_lddqu_si256 (in + 28); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 16)); - w1 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 24, w0); - tmp = _mm256_lddqu_si256 (in + 29); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 12)); - w0 = _mm256_srli_epi32(tmp,20); - _mm256_storeu_si256(compressed + 25, w1); - tmp = _mm256_lddqu_si256 (in + 30); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 8)); - w1 = _mm256_srli_epi32(tmp,24); - _mm256_storeu_si256(compressed + 26, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 31) , 4)); - _mm256_storeu_si256(compressed + 27, w1); -} - - -/* we are going to pack 256 29-bit values, touching 29 256-bit words, using 464 bytes */ -static void avxpackblock29(const uint32_t * pin, __m256i * compressed) { - const __m256i * in = (const __m256i *) pin; - /* we are going to touch 29 256-bit words */ - __m256i w0, w1; - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_lddqu_si256 (in + 0); - tmp = _mm256_lddqu_si256 (in + 1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 29)); - w1 = _mm256_srli_epi32(tmp,3); - _mm256_storeu_si256(compressed + 0, w0); - tmp = _mm256_lddqu_si256 (in + 2); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 26)); - w0 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 1, w1); - tmp = _mm256_lddqu_si256 (in + 3); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 23)); - w1 = _mm256_srli_epi32(tmp,9); - _mm256_storeu_si256(compressed + 2, w0); - tmp = _mm256_lddqu_si256 (in + 4); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 20)); - w0 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 3, w1); - tmp = _mm256_lddqu_si256 (in + 5); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 17)); - w1 = _mm256_srli_epi32(tmp,15); - _mm256_storeu_si256(compressed + 4, w0); - tmp = _mm256_lddqu_si256 (in + 6); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 14)); - w0 = _mm256_srli_epi32(tmp,18); - _mm256_storeu_si256(compressed + 5, w1); - tmp = _mm256_lddqu_si256 (in + 7); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 11)); - w1 = _mm256_srli_epi32(tmp,21); - _mm256_storeu_si256(compressed + 6, w0); - tmp = _mm256_lddqu_si256 (in + 8); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 8)); - w0 = _mm256_srli_epi32(tmp,24); - _mm256_storeu_si256(compressed + 7, w1); - tmp = _mm256_lddqu_si256 (in + 9); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 5)); - w1 = _mm256_srli_epi32(tmp,27); - _mm256_storeu_si256(compressed + 8, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 10) , 2)); - tmp = _mm256_lddqu_si256 (in + 11); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 31)); - w0 = _mm256_srli_epi32(tmp,1); - _mm256_storeu_si256(compressed + 9, w1); - tmp = _mm256_lddqu_si256 (in + 12); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 10, w0); - tmp = _mm256_lddqu_si256 (in + 13); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 25)); - w0 = _mm256_srli_epi32(tmp,7); - _mm256_storeu_si256(compressed + 11, w1); - tmp = _mm256_lddqu_si256 (in + 14); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 22)); - w1 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 12, w0); - tmp = _mm256_lddqu_si256 (in + 15); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 19)); - w0 = _mm256_srli_epi32(tmp,13); - _mm256_storeu_si256(compressed + 13, w1); - tmp = _mm256_lddqu_si256 (in + 16); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 16)); - w1 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 14, w0); - tmp = _mm256_lddqu_si256 (in + 17); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 13)); - w0 = _mm256_srli_epi32(tmp,19); - _mm256_storeu_si256(compressed + 15, w1); - tmp = _mm256_lddqu_si256 (in + 18); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 10)); - w1 = _mm256_srli_epi32(tmp,22); - _mm256_storeu_si256(compressed + 16, w0); - tmp = _mm256_lddqu_si256 (in + 19); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 7)); - w0 = _mm256_srli_epi32(tmp,25); - _mm256_storeu_si256(compressed + 17, w1); - tmp = _mm256_lddqu_si256 (in + 20); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 4)); - w1 = _mm256_srli_epi32(tmp,28); - _mm256_storeu_si256(compressed + 18, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 21) , 1)); - tmp = _mm256_lddqu_si256 (in + 22); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 30)); - w0 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 19, w1); - tmp = _mm256_lddqu_si256 (in + 23); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 27)); - w1 = _mm256_srli_epi32(tmp,5); - _mm256_storeu_si256(compressed + 20, w0); - tmp = _mm256_lddqu_si256 (in + 24); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 21, w1); - tmp = _mm256_lddqu_si256 (in + 25); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 21)); - w1 = _mm256_srli_epi32(tmp,11); - _mm256_storeu_si256(compressed + 22, w0); - tmp = _mm256_lddqu_si256 (in + 26); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 18)); - w0 = _mm256_srli_epi32(tmp,14); - _mm256_storeu_si256(compressed + 23, w1); - tmp = _mm256_lddqu_si256 (in + 27); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 15)); - w1 = _mm256_srli_epi32(tmp,17); - _mm256_storeu_si256(compressed + 24, w0); - tmp = _mm256_lddqu_si256 (in + 28); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 12)); - w0 = _mm256_srli_epi32(tmp,20); - _mm256_storeu_si256(compressed + 25, w1); - tmp = _mm256_lddqu_si256 (in + 29); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 9)); - w1 = _mm256_srli_epi32(tmp,23); - _mm256_storeu_si256(compressed + 26, w0); - tmp = _mm256_lddqu_si256 (in + 30); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 6)); - w0 = _mm256_srli_epi32(tmp,26); - _mm256_storeu_si256(compressed + 27, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 31) , 3)); - _mm256_storeu_si256(compressed + 28, w0); -} - - -/* we are going to pack 256 30-bit values, touching 30 256-bit words, using 480 bytes */ -static void avxpackblock30(const uint32_t * pin, __m256i * compressed) { - const __m256i * in = (const __m256i *) pin; - /* we are going to touch 30 256-bit words */ - __m256i w0, w1; - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_lddqu_si256 (in + 0); - tmp = _mm256_lddqu_si256 (in + 1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 30)); - w1 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 0, w0); - tmp = _mm256_lddqu_si256 (in + 2); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 1, w1); - tmp = _mm256_lddqu_si256 (in + 3); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 26)); - w1 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 2, w0); - tmp = _mm256_lddqu_si256 (in + 4); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 3, w1); - tmp = _mm256_lddqu_si256 (in + 5); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 22)); - w1 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 4, w0); - tmp = _mm256_lddqu_si256 (in + 6); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 20)); - w0 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 5, w1); - tmp = _mm256_lddqu_si256 (in + 7); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 18)); - w1 = _mm256_srli_epi32(tmp,14); - _mm256_storeu_si256(compressed + 6, w0); - tmp = _mm256_lddqu_si256 (in + 8); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 16)); - w0 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 7, w1); - tmp = _mm256_lddqu_si256 (in + 9); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 14)); - w1 = _mm256_srli_epi32(tmp,18); - _mm256_storeu_si256(compressed + 8, w0); - tmp = _mm256_lddqu_si256 (in + 10); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 12)); - w0 = _mm256_srli_epi32(tmp,20); - _mm256_storeu_si256(compressed + 9, w1); - tmp = _mm256_lddqu_si256 (in + 11); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 10)); - w1 = _mm256_srli_epi32(tmp,22); - _mm256_storeu_si256(compressed + 10, w0); - tmp = _mm256_lddqu_si256 (in + 12); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 8)); - w0 = _mm256_srli_epi32(tmp,24); - _mm256_storeu_si256(compressed + 11, w1); - tmp = _mm256_lddqu_si256 (in + 13); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 6)); - w1 = _mm256_srli_epi32(tmp,26); - _mm256_storeu_si256(compressed + 12, w0); - tmp = _mm256_lddqu_si256 (in + 14); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 4)); - w0 = _mm256_srli_epi32(tmp,28); - _mm256_storeu_si256(compressed + 13, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 15) , 2)); - _mm256_storeu_si256(compressed + 14, w0); - w1 = _mm256_lddqu_si256 (in + 16); - tmp = _mm256_lddqu_si256 (in + 17); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 30)); - w0 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 15, w1); - tmp = _mm256_lddqu_si256 (in + 18); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 16, w0); - tmp = _mm256_lddqu_si256 (in + 19); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 26)); - w0 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 17, w1); - tmp = _mm256_lddqu_si256 (in + 20); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 18, w0); - tmp = _mm256_lddqu_si256 (in + 21); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 22)); - w0 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 19, w1); - tmp = _mm256_lddqu_si256 (in + 22); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 20)); - w1 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 20, w0); - tmp = _mm256_lddqu_si256 (in + 23); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 18)); - w0 = _mm256_srli_epi32(tmp,14); - _mm256_storeu_si256(compressed + 21, w1); - tmp = _mm256_lddqu_si256 (in + 24); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 16)); - w1 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 22, w0); - tmp = _mm256_lddqu_si256 (in + 25); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 14)); - w0 = _mm256_srli_epi32(tmp,18); - _mm256_storeu_si256(compressed + 23, w1); - tmp = _mm256_lddqu_si256 (in + 26); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 12)); - w1 = _mm256_srli_epi32(tmp,20); - _mm256_storeu_si256(compressed + 24, w0); - tmp = _mm256_lddqu_si256 (in + 27); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 10)); - w0 = _mm256_srli_epi32(tmp,22); - _mm256_storeu_si256(compressed + 25, w1); - tmp = _mm256_lddqu_si256 (in + 28); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 8)); - w1 = _mm256_srli_epi32(tmp,24); - _mm256_storeu_si256(compressed + 26, w0); - tmp = _mm256_lddqu_si256 (in + 29); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 6)); - w0 = _mm256_srli_epi32(tmp,26); - _mm256_storeu_si256(compressed + 27, w1); - tmp = _mm256_lddqu_si256 (in + 30); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 4)); - w1 = _mm256_srli_epi32(tmp,28); - _mm256_storeu_si256(compressed + 28, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 31) , 2)); - _mm256_storeu_si256(compressed + 29, w1); -} - - -/* we are going to pack 256 31-bit values, touching 31 256-bit words, using 496 bytes */ -static void avxpackblock31(const uint32_t * pin, __m256i * compressed) { - const __m256i * in = (const __m256i *) pin; - /* we are going to touch 31 256-bit words */ - __m256i w0, w1; - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_lddqu_si256 (in + 0); - tmp = _mm256_lddqu_si256 (in + 1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 31)); - w1 = _mm256_srli_epi32(tmp,1); - _mm256_storeu_si256(compressed + 0, w0); - tmp = _mm256_lddqu_si256 (in + 2); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 30)); - w0 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 1, w1); - tmp = _mm256_lddqu_si256 (in + 3); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 29)); - w1 = _mm256_srli_epi32(tmp,3); - _mm256_storeu_si256(compressed + 2, w0); - tmp = _mm256_lddqu_si256 (in + 4); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 3, w1); - tmp = _mm256_lddqu_si256 (in + 5); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 27)); - w1 = _mm256_srli_epi32(tmp,5); - _mm256_storeu_si256(compressed + 4, w0); - tmp = _mm256_lddqu_si256 (in + 6); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 26)); - w0 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 5, w1); - tmp = _mm256_lddqu_si256 (in + 7); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 25)); - w1 = _mm256_srli_epi32(tmp,7); - _mm256_storeu_si256(compressed + 6, w0); - tmp = _mm256_lddqu_si256 (in + 8); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 7, w1); - tmp = _mm256_lddqu_si256 (in + 9); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 23)); - w1 = _mm256_srli_epi32(tmp,9); - _mm256_storeu_si256(compressed + 8, w0); - tmp = _mm256_lddqu_si256 (in + 10); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 22)); - w0 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 9, w1); - tmp = _mm256_lddqu_si256 (in + 11); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 21)); - w1 = _mm256_srli_epi32(tmp,11); - _mm256_storeu_si256(compressed + 10, w0); - tmp = _mm256_lddqu_si256 (in + 12); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 20)); - w0 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 11, w1); - tmp = _mm256_lddqu_si256 (in + 13); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 19)); - w1 = _mm256_srli_epi32(tmp,13); - _mm256_storeu_si256(compressed + 12, w0); - tmp = _mm256_lddqu_si256 (in + 14); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 18)); - w0 = _mm256_srli_epi32(tmp,14); - _mm256_storeu_si256(compressed + 13, w1); - tmp = _mm256_lddqu_si256 (in + 15); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 17)); - w1 = _mm256_srli_epi32(tmp,15); - _mm256_storeu_si256(compressed + 14, w0); - tmp = _mm256_lddqu_si256 (in + 16); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 16)); - w0 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 15, w1); - tmp = _mm256_lddqu_si256 (in + 17); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 15)); - w1 = _mm256_srli_epi32(tmp,17); - _mm256_storeu_si256(compressed + 16, w0); - tmp = _mm256_lddqu_si256 (in + 18); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 14)); - w0 = _mm256_srli_epi32(tmp,18); - _mm256_storeu_si256(compressed + 17, w1); - tmp = _mm256_lddqu_si256 (in + 19); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 13)); - w1 = _mm256_srli_epi32(tmp,19); - _mm256_storeu_si256(compressed + 18, w0); - tmp = _mm256_lddqu_si256 (in + 20); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 12)); - w0 = _mm256_srli_epi32(tmp,20); - _mm256_storeu_si256(compressed + 19, w1); - tmp = _mm256_lddqu_si256 (in + 21); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 11)); - w1 = _mm256_srli_epi32(tmp,21); - _mm256_storeu_si256(compressed + 20, w0); - tmp = _mm256_lddqu_si256 (in + 22); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 10)); - w0 = _mm256_srli_epi32(tmp,22); - _mm256_storeu_si256(compressed + 21, w1); - tmp = _mm256_lddqu_si256 (in + 23); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 9)); - w1 = _mm256_srli_epi32(tmp,23); - _mm256_storeu_si256(compressed + 22, w0); - tmp = _mm256_lddqu_si256 (in + 24); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 8)); - w0 = _mm256_srli_epi32(tmp,24); - _mm256_storeu_si256(compressed + 23, w1); - tmp = _mm256_lddqu_si256 (in + 25); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 7)); - w1 = _mm256_srli_epi32(tmp,25); - _mm256_storeu_si256(compressed + 24, w0); - tmp = _mm256_lddqu_si256 (in + 26); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 6)); - w0 = _mm256_srli_epi32(tmp,26); - _mm256_storeu_si256(compressed + 25, w1); - tmp = _mm256_lddqu_si256 (in + 27); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 5)); - w1 = _mm256_srli_epi32(tmp,27); - _mm256_storeu_si256(compressed + 26, w0); - tmp = _mm256_lddqu_si256 (in + 28); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 4)); - w0 = _mm256_srli_epi32(tmp,28); - _mm256_storeu_si256(compressed + 27, w1); - tmp = _mm256_lddqu_si256 (in + 29); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 3)); - w1 = _mm256_srli_epi32(tmp,29); - _mm256_storeu_si256(compressed + 28, w0); - tmp = _mm256_lddqu_si256 (in + 30); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 2)); - w0 = _mm256_srli_epi32(tmp,30); - _mm256_storeu_si256(compressed + 29, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(_mm256_lddqu_si256 (in + 31) , 1)); - _mm256_storeu_si256(compressed + 30, w0); -} - - -/* we are going to pack 256 32-bit values, touching 32 256-bit words, using 512 bytes */ -static void avxpackblock32(const uint32_t * pin, __m256i * compressed) { - const __m256i * in = (const __m256i *) pin; - /* we are going to touch 32 256-bit words */ - __m256i w0, w1; - w0 = _mm256_lddqu_si256 (in + 0); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_lddqu_si256 (in + 1); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_lddqu_si256 (in + 2); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_lddqu_si256 (in + 3); - _mm256_storeu_si256(compressed + 3, w1); - w0 = _mm256_lddqu_si256 (in + 4); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_lddqu_si256 (in + 5); - _mm256_storeu_si256(compressed + 5, w1); - w0 = _mm256_lddqu_si256 (in + 6); - _mm256_storeu_si256(compressed + 6, w0); - w1 = _mm256_lddqu_si256 (in + 7); - _mm256_storeu_si256(compressed + 7, w1); - w0 = _mm256_lddqu_si256 (in + 8); - _mm256_storeu_si256(compressed + 8, w0); - w1 = _mm256_lddqu_si256 (in + 9); - _mm256_storeu_si256(compressed + 9, w1); - w0 = _mm256_lddqu_si256 (in + 10); - _mm256_storeu_si256(compressed + 10, w0); - w1 = _mm256_lddqu_si256 (in + 11); - _mm256_storeu_si256(compressed + 11, w1); - w0 = _mm256_lddqu_si256 (in + 12); - _mm256_storeu_si256(compressed + 12, w0); - w1 = _mm256_lddqu_si256 (in + 13); - _mm256_storeu_si256(compressed + 13, w1); - w0 = _mm256_lddqu_si256 (in + 14); - _mm256_storeu_si256(compressed + 14, w0); - w1 = _mm256_lddqu_si256 (in + 15); - _mm256_storeu_si256(compressed + 15, w1); - w0 = _mm256_lddqu_si256 (in + 16); - _mm256_storeu_si256(compressed + 16, w0); - w1 = _mm256_lddqu_si256 (in + 17); - _mm256_storeu_si256(compressed + 17, w1); - w0 = _mm256_lddqu_si256 (in + 18); - _mm256_storeu_si256(compressed + 18, w0); - w1 = _mm256_lddqu_si256 (in + 19); - _mm256_storeu_si256(compressed + 19, w1); - w0 = _mm256_lddqu_si256 (in + 20); - _mm256_storeu_si256(compressed + 20, w0); - w1 = _mm256_lddqu_si256 (in + 21); - _mm256_storeu_si256(compressed + 21, w1); - w0 = _mm256_lddqu_si256 (in + 22); - _mm256_storeu_si256(compressed + 22, w0); - w1 = _mm256_lddqu_si256 (in + 23); - _mm256_storeu_si256(compressed + 23, w1); - w0 = _mm256_lddqu_si256 (in + 24); - _mm256_storeu_si256(compressed + 24, w0); - w1 = _mm256_lddqu_si256 (in + 25); - _mm256_storeu_si256(compressed + 25, w1); - w0 = _mm256_lddqu_si256 (in + 26); - _mm256_storeu_si256(compressed + 26, w0); - w1 = _mm256_lddqu_si256 (in + 27); - _mm256_storeu_si256(compressed + 27, w1); - w0 = _mm256_lddqu_si256 (in + 28); - _mm256_storeu_si256(compressed + 28, w0); - w1 = _mm256_lddqu_si256 (in + 29); - _mm256_storeu_si256(compressed + 29, w1); - w0 = _mm256_lddqu_si256 (in + 30); - _mm256_storeu_si256(compressed + 30, w0); - w1 = _mm256_lddqu_si256 (in + 31); - _mm256_storeu_si256(compressed + 31, w1); -} - - -static void avxpackblockmask0(const uint32_t * pin, __m256i * compressed) { - (void)compressed; - (void) pin; /* we consumed 256 32-bit integers */ -} - - -/* we are going to pack 256 1-bit values, touching 1 256-bit words, using 16 bytes */ -static void avxpackblockmask1(const uint32_t * pin, __m256i * compressed) { - /* we are going to touch 1 256-bit word */ - __m256i w0; - const __m256i * in = (const __m256i *) pin; - const __m256i mask = _mm256_set1_epi32(1); - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 0) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 1) ) , 1)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 2) ) , 2)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 3) ) , 3)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 4) ) , 4)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 5) ) , 5)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 6) ) , 6)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 7) ) , 7)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 8) ) , 8)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 9) ) , 9)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 10) ) , 10)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 11) ) , 11)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 12) ) , 12)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 13) ) , 13)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 14) ) , 14)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 15) ) , 15)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 16) ) , 16)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 17) ) , 17)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 18) ) , 18)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 19) ) , 19)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 20) ) , 20)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 21) ) , 21)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 22) ) , 22)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 23) ) , 23)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 24) ) , 24)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 25) ) , 25)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 26) ) , 26)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 27) ) , 27)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 28) ) , 28)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 29) ) , 29)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 30) ) , 30)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 31) ) , 31)); - _mm256_storeu_si256(compressed + 0, w0); -} - - -/* we are going to pack 256 2-bit values, touching 2 256-bit words, using 32 bytes */ -static void avxpackblockmask2(const uint32_t * pin, __m256i * compressed) { - /* we are going to touch 2 256-bit words */ - __m256i w0, w1; - const __m256i * in = (const __m256i *) pin; - const __m256i mask = _mm256_set1_epi32(3); - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 0) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 1) ) , 2)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 2) ) , 4)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 3) ) , 6)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 4) ) , 8)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 5) ) , 10)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 6) ) , 12)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 7) ) , 14)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 8) ) , 16)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 9) ) , 18)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 10) ) , 20)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 11) ) , 22)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 12) ) , 24)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 13) ) , 26)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 14) ) , 28)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 15) ) , 30)); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 16) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 17) ) , 2)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 18) ) , 4)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 19) ) , 6)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 20) ) , 8)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 21) ) , 10)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 22) ) , 12)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 23) ) , 14)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 24) ) , 16)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 25) ) , 18)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 26) ) , 20)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 27) ) , 22)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 28) ) , 24)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 29) ) , 26)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 30) ) , 28)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 31) ) , 30)); - _mm256_storeu_si256(compressed + 1, w1); -} - - -/* we are going to pack 256 3-bit values, touching 3 256-bit words, using 48 bytes */ -static void avxpackblockmask3(const uint32_t * pin, __m256i * compressed) { - /* we are going to touch 3 256-bit words */ - __m256i w0, w1; - const __m256i * in = (const __m256i *) pin; - const __m256i mask = _mm256_set1_epi32(7); - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 0) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 1) ) , 3)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 2) ) , 6)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 3) ) , 9)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 4) ) , 12)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 5) ) , 15)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 6) ) , 18)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 7) ) , 21)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 8) ) , 24)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 9) ) , 27)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 10) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 30)); - w1 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 11) ) , 1)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 12) ) , 4)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 13) ) , 7)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 14) ) , 10)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 15) ) , 13)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 16) ) , 16)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 17) ) , 19)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 18) ) , 22)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 19) ) , 25)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 20) ) , 28)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 21) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 31)); - w0 = _mm256_srli_epi32(tmp,1); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 22) ) , 2)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 23) ) , 5)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 24) ) , 8)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 25) ) , 11)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 26) ) , 14)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 27) ) , 17)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 28) ) , 20)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 29) ) , 23)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 30) ) , 26)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 31) ) , 29)); - _mm256_storeu_si256(compressed + 2, w0); -} - - -/* we are going to pack 256 4-bit values, touching 4 256-bit words, using 64 bytes */ -static void avxpackblockmask4(const uint32_t * pin, __m256i * compressed) { - /* we are going to touch 4 256-bit words */ - __m256i w0, w1; - const __m256i * in = (const __m256i *) pin; - const __m256i mask = _mm256_set1_epi32(15); - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 0) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 1) ) , 4)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 2) ) , 8)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 3) ) , 12)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 4) ) , 16)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 5) ) , 20)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 6) ) , 24)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 7) ) , 28)); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 8) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 9) ) , 4)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 10) ) , 8)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 11) ) , 12)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 12) ) , 16)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 13) ) , 20)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 14) ) , 24)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 15) ) , 28)); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 16) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 17) ) , 4)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 18) ) , 8)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 19) ) , 12)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 20) ) , 16)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 21) ) , 20)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 22) ) , 24)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 23) ) , 28)); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 24) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 25) ) , 4)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 26) ) , 8)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 27) ) , 12)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 28) ) , 16)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 29) ) , 20)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 30) ) , 24)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 31) ) , 28)); - _mm256_storeu_si256(compressed + 3, w1); -} - - -/* we are going to pack 256 5-bit values, touching 5 256-bit words, using 80 bytes */ -static void avxpackblockmask5(const uint32_t * pin, __m256i * compressed) { - /* we are going to touch 5 256-bit words */ - __m256i w0, w1; - const __m256i * in = (const __m256i *) pin; - const __m256i mask = _mm256_set1_epi32(31); - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 0) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 1) ) , 5)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 2) ) , 10)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 3) ) , 15)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 4) ) , 20)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 5) ) , 25)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 6) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 30)); - w1 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 7) ) , 3)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 8) ) , 8)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 9) ) , 13)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 10) ) , 18)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 11) ) , 23)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 12) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 13) ) , 1)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 14) ) , 6)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 15) ) , 11)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 16) ) , 16)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 17) ) , 21)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 18) ) , 26)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 19) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 31)); - w1 = _mm256_srli_epi32(tmp,1); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 20) ) , 4)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 21) ) , 9)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 22) ) , 14)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 23) ) , 19)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 24) ) , 24)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 25) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 29)); - w0 = _mm256_srli_epi32(tmp,3); - _mm256_storeu_si256(compressed + 3, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 26) ) , 2)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 27) ) , 7)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 28) ) , 12)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 29) ) , 17)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 30) ) , 22)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 31) ) , 27)); - _mm256_storeu_si256(compressed + 4, w0); -} - - -/* we are going to pack 256 6-bit values, touching 6 256-bit words, using 96 bytes */ -static void avxpackblockmask6(const uint32_t * pin, __m256i * compressed) { - /* we are going to touch 6 256-bit words */ - __m256i w0, w1; - const __m256i * in = (const __m256i *) pin; - const __m256i mask = _mm256_set1_epi32(63); - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 0) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 1) ) , 6)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 2) ) , 12)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 3) ) , 18)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 4) ) , 24)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 5) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 30)); - w1 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 6) ) , 4)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 7) ) , 10)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 8) ) , 16)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 9) ) , 22)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 10) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 11) ) , 2)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 12) ) , 8)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 13) ) , 14)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 14) ) , 20)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 15) ) , 26)); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 16) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 17) ) , 6)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 18) ) , 12)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 19) ) , 18)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 20) ) , 24)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 21) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 30)); - w0 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 3, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 22) ) , 4)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 23) ) , 10)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 24) ) , 16)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 25) ) , 22)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 26) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 27) ) , 2)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 28) ) , 8)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 29) ) , 14)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 30) ) , 20)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 31) ) , 26)); - _mm256_storeu_si256(compressed + 5, w1); -} - - -/* we are going to pack 256 7-bit values, touching 7 256-bit words, using 112 bytes */ -static void avxpackblockmask7(const uint32_t * pin, __m256i * compressed) { - /* we are going to touch 7 256-bit words */ - __m256i w0, w1; - const __m256i * in = (const __m256i *) pin; - const __m256i mask = _mm256_set1_epi32(127); - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 0) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 1) ) , 7)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 2) ) , 14)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 3) ) , 21)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 4) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 5) ) , 3)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 6) ) , 10)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 7) ) , 17)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 8) ) , 24)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 9) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 31)); - w0 = _mm256_srli_epi32(tmp,1); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 10) ) , 6)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 11) ) , 13)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 12) ) , 20)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 13) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 27)); - w1 = _mm256_srli_epi32(tmp,5); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 14) ) , 2)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 15) ) , 9)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 16) ) , 16)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 17) ) , 23)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 18) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 30)); - w0 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 3, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 19) ) , 5)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 20) ) , 12)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 21) ) , 19)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 22) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 26)); - w1 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 23) ) , 1)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 24) ) , 8)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 25) ) , 15)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 26) ) , 22)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 27) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 29)); - w0 = _mm256_srli_epi32(tmp,3); - _mm256_storeu_si256(compressed + 5, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 28) ) , 4)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 29) ) , 11)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 30) ) , 18)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 31) ) , 25)); - _mm256_storeu_si256(compressed + 6, w0); -} - - -/* we are going to pack 256 8-bit values, touching 8 256-bit words, using 128 bytes */ -static void avxpackblockmask8(const uint32_t * pin, __m256i * compressed) { - /* we are going to touch 8 256-bit words */ - __m256i w0, w1; - const __m256i * in = (const __m256i *) pin; - const __m256i mask = _mm256_set1_epi32(255); - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 0) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 1) ) , 8)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 2) ) , 16)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 3) ) , 24)); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 4) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 5) ) , 8)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 6) ) , 16)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 7) ) , 24)); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 8) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 9) ) , 8)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 10) ) , 16)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 11) ) , 24)); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 12) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 13) ) , 8)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 14) ) , 16)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 15) ) , 24)); - _mm256_storeu_si256(compressed + 3, w1); - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 16) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 17) ) , 8)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 18) ) , 16)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 19) ) , 24)); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 20) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 21) ) , 8)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 22) ) , 16)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 23) ) , 24)); - _mm256_storeu_si256(compressed + 5, w1); - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 24) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 25) ) , 8)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 26) ) , 16)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 27) ) , 24)); - _mm256_storeu_si256(compressed + 6, w0); - w1 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 28) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 29) ) , 8)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 30) ) , 16)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 31) ) , 24)); - _mm256_storeu_si256(compressed + 7, w1); -} - - -/* we are going to pack 256 9-bit values, touching 9 256-bit words, using 144 bytes */ -static void avxpackblockmask9(const uint32_t * pin, __m256i * compressed) { - /* we are going to touch 9 256-bit words */ - __m256i w0, w1; - const __m256i * in = (const __m256i *) pin; - const __m256i mask = _mm256_set1_epi32(511); - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 0) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 1) ) , 9)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 2) ) , 18)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 3) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 27)); - w1 = _mm256_srli_epi32(tmp,5); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 4) ) , 4)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 5) ) , 13)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 6) ) , 22)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 7) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 31)); - w0 = _mm256_srli_epi32(tmp,1); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 8) ) , 8)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 9) ) , 17)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 10) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 26)); - w1 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 11) ) , 3)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 12) ) , 12)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 13) ) , 21)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 14) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 30)); - w0 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 3, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 15) ) , 7)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 16) ) , 16)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 17) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 25)); - w1 = _mm256_srli_epi32(tmp,7); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 18) ) , 2)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 19) ) , 11)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 20) ) , 20)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 21) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 29)); - w0 = _mm256_srli_epi32(tmp,3); - _mm256_storeu_si256(compressed + 5, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 22) ) , 6)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 23) ) , 15)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 24) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 6, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 25) ) , 1)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 26) ) , 10)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 27) ) , 19)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 28) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 7, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 29) ) , 5)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 30) ) , 14)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 31) ) , 23)); - _mm256_storeu_si256(compressed + 8, w0); -} - - -/* we are going to pack 256 10-bit values, touching 10 256-bit words, using 160 bytes */ -static void avxpackblockmask10(const uint32_t * pin, __m256i * compressed) { - /* we are going to touch 10 256-bit words */ - __m256i w0, w1; - const __m256i * in = (const __m256i *) pin; - const __m256i mask = _mm256_set1_epi32(1023); - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 0) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 1) ) , 10)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 2) ) , 20)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 3) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 30)); - w1 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 4) ) , 8)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 5) ) , 18)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 6) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 7) ) , 6)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 8) ) , 16)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 9) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 26)); - w1 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 10) ) , 4)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 11) ) , 14)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 12) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 3, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 13) ) , 2)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 14) ) , 12)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 15) ) , 22)); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 16) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 17) ) , 10)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 18) ) , 20)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 19) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 30)); - w0 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 5, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 20) ) , 8)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 21) ) , 18)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 22) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 6, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 23) ) , 6)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 24) ) , 16)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 25) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 26)); - w0 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 7, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 26) ) , 4)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 27) ) , 14)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 28) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 8, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 29) ) , 2)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 30) ) , 12)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 31) ) , 22)); - _mm256_storeu_si256(compressed + 9, w1); -} - - -/* we are going to pack 256 11-bit values, touching 11 256-bit words, using 176 bytes */ -static void avxpackblockmask11(const uint32_t * pin, __m256i * compressed) { - /* we are going to touch 11 256-bit words */ - __m256i w0, w1; - const __m256i * in = (const __m256i *) pin; - const __m256i mask = _mm256_set1_epi32(2047); - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 0) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 1) ) , 11)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 2) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 22)); - w1 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 3) ) , 1)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 4) ) , 12)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 5) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 23)); - w0 = _mm256_srli_epi32(tmp,9); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 6) ) , 2)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 7) ) , 13)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 8) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 9) ) , 3)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 10) ) , 14)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 11) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 25)); - w0 = _mm256_srli_epi32(tmp,7); - _mm256_storeu_si256(compressed + 3, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 12) ) , 4)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 13) ) , 15)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 14) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 26)); - w1 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 15) ) , 5)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 16) ) , 16)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 17) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 27)); - w0 = _mm256_srli_epi32(tmp,5); - _mm256_storeu_si256(compressed + 5, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 18) ) , 6)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 19) ) , 17)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 20) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 6, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 21) ) , 7)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 22) ) , 18)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 23) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 29)); - w0 = _mm256_srli_epi32(tmp,3); - _mm256_storeu_si256(compressed + 7, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 24) ) , 8)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 25) ) , 19)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 26) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 30)); - w1 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 8, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 27) ) , 9)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 28) ) , 20)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 29) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 31)); - w0 = _mm256_srli_epi32(tmp,1); - _mm256_storeu_si256(compressed + 9, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 30) ) , 10)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 31) ) , 21)); - _mm256_storeu_si256(compressed + 10, w0); -} - - -/* we are going to pack 256 12-bit values, touching 12 256-bit words, using 192 bytes */ -static void avxpackblockmask12(const uint32_t * pin, __m256i * compressed) { - /* we are going to touch 12 256-bit words */ - __m256i w0, w1; - const __m256i * in = (const __m256i *) pin; - const __m256i mask = _mm256_set1_epi32(4095); - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 0) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 1) ) , 12)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 2) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 3) ) , 4)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 4) ) , 16)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 5) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 6) ) , 8)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 7) ) , 20)); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 8) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 9) ) , 12)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 10) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 3, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 11) ) , 4)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 12) ) , 16)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 13) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 14) ) , 8)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 15) ) , 20)); - _mm256_storeu_si256(compressed + 5, w1); - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 16) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 17) ) , 12)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 18) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 6, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 19) ) , 4)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 20) ) , 16)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 21) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 7, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 22) ) , 8)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 23) ) , 20)); - _mm256_storeu_si256(compressed + 8, w0); - w1 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 24) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 25) ) , 12)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 26) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 9, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 27) ) , 4)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 28) ) , 16)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 29) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 10, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 30) ) , 8)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 31) ) , 20)); - _mm256_storeu_si256(compressed + 11, w1); -} - - -/* we are going to pack 256 13-bit values, touching 13 256-bit words, using 208 bytes */ -static void avxpackblockmask13(const uint32_t * pin, __m256i * compressed) { - /* we are going to touch 13 256-bit words */ - __m256i w0, w1; - const __m256i * in = (const __m256i *) pin; - const __m256i mask = _mm256_set1_epi32(8191); - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 0) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 1) ) , 13)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 2) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 26)); - w1 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 3) ) , 7)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 4) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 20)); - w0 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 5) ) , 1)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 6) ) , 14)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 7) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 27)); - w1 = _mm256_srli_epi32(tmp,5); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 8) ) , 8)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 9) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 21)); - w0 = _mm256_srli_epi32(tmp,11); - _mm256_storeu_si256(compressed + 3, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 10) ) , 2)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 11) ) , 15)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 12) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 13) ) , 9)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 14) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 22)); - w0 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 5, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 15) ) , 3)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 16) ) , 16)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 17) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 29)); - w1 = _mm256_srli_epi32(tmp,3); - _mm256_storeu_si256(compressed + 6, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 18) ) , 10)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 19) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 23)); - w0 = _mm256_srli_epi32(tmp,9); - _mm256_storeu_si256(compressed + 7, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 20) ) , 4)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 21) ) , 17)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 22) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 30)); - w1 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 8, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 23) ) , 11)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 24) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 9, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 25) ) , 5)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 26) ) , 18)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 27) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 31)); - w1 = _mm256_srli_epi32(tmp,1); - _mm256_storeu_si256(compressed + 10, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 28) ) , 12)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 29) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 25)); - w0 = _mm256_srli_epi32(tmp,7); - _mm256_storeu_si256(compressed + 11, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 30) ) , 6)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 31) ) , 19)); - _mm256_storeu_si256(compressed + 12, w0); -} - - -/* we are going to pack 256 14-bit values, touching 14 256-bit words, using 224 bytes */ -static void avxpackblockmask14(const uint32_t * pin, __m256i * compressed) { - /* we are going to touch 14 256-bit words */ - __m256i w0, w1; - const __m256i * in = (const __m256i *) pin; - const __m256i mask = _mm256_set1_epi32(16383); - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 0) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 1) ) , 14)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 2) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 3) ) , 10)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 4) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 5) ) , 6)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 6) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 20)); - w1 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 7) ) , 2)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 8) ) , 16)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 9) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 30)); - w0 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 3, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 10) ) , 12)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 11) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 26)); - w1 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 12) ) , 8)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 13) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 22)); - w0 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 5, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 14) ) , 4)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 15) ) , 18)); - _mm256_storeu_si256(compressed + 6, w0); - w1 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 16) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 17) ) , 14)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 18) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 7, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 19) ) , 10)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 20) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 8, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 21) ) , 6)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 22) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 20)); - w0 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 9, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 23) ) , 2)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 24) ) , 16)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 25) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 30)); - w1 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 10, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 26) ) , 12)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 27) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 26)); - w0 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 11, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 28) ) , 8)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 29) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 22)); - w1 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 12, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 30) ) , 4)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 31) ) , 18)); - _mm256_storeu_si256(compressed + 13, w1); -} - - -/* we are going to pack 256 15-bit values, touching 15 256-bit words, using 240 bytes */ -static void avxpackblockmask15(const uint32_t * pin, __m256i * compressed) { - /* we are going to touch 15 256-bit words */ - __m256i w0, w1; - const __m256i * in = (const __m256i *) pin; - const __m256i mask = _mm256_set1_epi32(32767); - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 0) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 1) ) , 15)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 2) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 30)); - w1 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 3) ) , 13)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 4) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 5) ) , 11)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 6) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 26)); - w1 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 7) ) , 9)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 8) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 3, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 9) ) , 7)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 10) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 22)); - w1 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 11) ) , 5)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 12) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 20)); - w0 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 5, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 13) ) , 3)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 14) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 18)); - w1 = _mm256_srli_epi32(tmp,14); - _mm256_storeu_si256(compressed + 6, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 15) ) , 1)); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 16) ) , 16)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 17) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 31)); - w0 = _mm256_srli_epi32(tmp,1); - _mm256_storeu_si256(compressed + 7, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 18) ) , 14)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 19) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 29)); - w1 = _mm256_srli_epi32(tmp,3); - _mm256_storeu_si256(compressed + 8, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 20) ) , 12)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 21) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 27)); - w0 = _mm256_srli_epi32(tmp,5); - _mm256_storeu_si256(compressed + 9, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 22) ) , 10)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 23) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 25)); - w1 = _mm256_srli_epi32(tmp,7); - _mm256_storeu_si256(compressed + 10, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 24) ) , 8)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 25) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 23)); - w0 = _mm256_srli_epi32(tmp,9); - _mm256_storeu_si256(compressed + 11, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 26) ) , 6)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 27) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 21)); - w1 = _mm256_srli_epi32(tmp,11); - _mm256_storeu_si256(compressed + 12, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 28) ) , 4)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 29) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 19)); - w0 = _mm256_srli_epi32(tmp,13); - _mm256_storeu_si256(compressed + 13, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 30) ) , 2)); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 31) ) , 17)); - _mm256_storeu_si256(compressed + 14, w0); -} - - -/* we are going to pack 256 16-bit values, touching 16 256-bit words, using 256 bytes */ -static void avxpackblockmask16(const uint32_t * pin, __m256i * compressed) { - /* we are going to touch 16 256-bit words */ - __m256i w0, w1; - const __m256i * in = (const __m256i *) pin; - const __m256i mask = _mm256_set1_epi32(65535); - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 0) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 1) ) , 16)); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 2) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 3) ) , 16)); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 4) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 5) ) , 16)); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 6) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 7) ) , 16)); - _mm256_storeu_si256(compressed + 3, w1); - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 8) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 9) ) , 16)); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 10) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 11) ) , 16)); - _mm256_storeu_si256(compressed + 5, w1); - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 12) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 13) ) , 16)); - _mm256_storeu_si256(compressed + 6, w0); - w1 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 14) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 15) ) , 16)); - _mm256_storeu_si256(compressed + 7, w1); - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 16) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 17) ) , 16)); - _mm256_storeu_si256(compressed + 8, w0); - w1 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 18) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 19) ) , 16)); - _mm256_storeu_si256(compressed + 9, w1); - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 20) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 21) ) , 16)); - _mm256_storeu_si256(compressed + 10, w0); - w1 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 22) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 23) ) , 16)); - _mm256_storeu_si256(compressed + 11, w1); - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 24) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 25) ) , 16)); - _mm256_storeu_si256(compressed + 12, w0); - w1 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 26) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 27) ) , 16)); - _mm256_storeu_si256(compressed + 13, w1); - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 28) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 29) ) , 16)); - _mm256_storeu_si256(compressed + 14, w0); - w1 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 30) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 31) ) , 16)); - _mm256_storeu_si256(compressed + 15, w1); -} - - -/* we are going to pack 256 17-bit values, touching 17 256-bit words, using 272 bytes */ -static void avxpackblockmask17(const uint32_t * pin, __m256i * compressed) { - /* we are going to touch 17 256-bit words */ - __m256i w0, w1; - const __m256i * in = (const __m256i *) pin; - const __m256i mask = _mm256_set1_epi32(131071); - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 0) ) ; - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 1) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 17)); - w1 = _mm256_srli_epi32(tmp,15); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 2) ) , 2)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 3) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 19)); - w0 = _mm256_srli_epi32(tmp,13); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 4) ) , 4)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 5) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 21)); - w1 = _mm256_srli_epi32(tmp,11); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 6) ) , 6)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 7) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 23)); - w0 = _mm256_srli_epi32(tmp,9); - _mm256_storeu_si256(compressed + 3, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 8) ) , 8)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 9) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 25)); - w1 = _mm256_srli_epi32(tmp,7); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 10) ) , 10)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 11) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 27)); - w0 = _mm256_srli_epi32(tmp,5); - _mm256_storeu_si256(compressed + 5, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 12) ) , 12)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 13) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 29)); - w1 = _mm256_srli_epi32(tmp,3); - _mm256_storeu_si256(compressed + 6, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 14) ) , 14)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 15) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 31)); - w0 = _mm256_srli_epi32(tmp,1); - _mm256_storeu_si256(compressed + 7, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 16) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 16)); - w1 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 8, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 17) ) , 1)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 18) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 18)); - w0 = _mm256_srli_epi32(tmp,14); - _mm256_storeu_si256(compressed + 9, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 19) ) , 3)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 20) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 20)); - w1 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 10, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 21) ) , 5)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 22) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 22)); - w0 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 11, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 23) ) , 7)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 24) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 12, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 25) ) , 9)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 26) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 26)); - w0 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 13, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 27) ) , 11)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 28) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 14, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 29) ) , 13)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 30) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 30)); - w0 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 15, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 31) ) , 15)); - _mm256_storeu_si256(compressed + 16, w0); -} - - -/* we are going to pack 256 18-bit values, touching 18 256-bit words, using 288 bytes */ -static void avxpackblockmask18(const uint32_t * pin, __m256i * compressed) { - /* we are going to touch 18 256-bit words */ - __m256i w0, w1; - const __m256i * in = (const __m256i *) pin; - const __m256i mask = _mm256_set1_epi32(262143); - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 0) ) ; - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 1) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 18)); - w1 = _mm256_srli_epi32(tmp,14); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 2) ) , 4)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 3) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 22)); - w0 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 4) ) , 8)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 5) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 26)); - w1 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 6) ) , 12)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 7) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 30)); - w0 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 3, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 8) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 16)); - w1 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 9) ) , 2)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 10) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 20)); - w0 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 5, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 11) ) , 6)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 12) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 6, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 13) ) , 10)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 14) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 7, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 15) ) , 14)); - _mm256_storeu_si256(compressed + 8, w0); - w1 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 16) ) ; - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 17) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 18)); - w0 = _mm256_srli_epi32(tmp,14); - _mm256_storeu_si256(compressed + 9, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 18) ) , 4)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 19) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 22)); - w1 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 10, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 20) ) , 8)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 21) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 26)); - w0 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 11, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 22) ) , 12)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 23) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 30)); - w1 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 12, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 24) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 16)); - w0 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 13, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 25) ) , 2)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 26) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 20)); - w1 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 14, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 27) ) , 6)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 28) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 15, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 29) ) , 10)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 30) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 16, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 31) ) , 14)); - _mm256_storeu_si256(compressed + 17, w1); -} - - -/* we are going to pack 256 19-bit values, touching 19 256-bit words, using 304 bytes */ -static void avxpackblockmask19(const uint32_t * pin, __m256i * compressed) { - /* we are going to touch 19 256-bit words */ - __m256i w0, w1; - const __m256i * in = (const __m256i *) pin; - const __m256i mask = _mm256_set1_epi32(524287); - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 0) ) ; - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 1) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 19)); - w1 = _mm256_srli_epi32(tmp,13); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 2) ) , 6)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 3) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 25)); - w0 = _mm256_srli_epi32(tmp,7); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 4) ) , 12)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 5) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 31)); - w1 = _mm256_srli_epi32(tmp,1); - _mm256_storeu_si256(compressed + 2, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 6) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 18)); - w0 = _mm256_srli_epi32(tmp,14); - _mm256_storeu_si256(compressed + 3, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 7) ) , 5)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 8) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 9) ) , 11)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 10) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 30)); - w0 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 5, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 11) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 17)); - w1 = _mm256_srli_epi32(tmp,15); - _mm256_storeu_si256(compressed + 6, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 12) ) , 4)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 13) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 23)); - w0 = _mm256_srli_epi32(tmp,9); - _mm256_storeu_si256(compressed + 7, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 14) ) , 10)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 15) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 29)); - w1 = _mm256_srli_epi32(tmp,3); - _mm256_storeu_si256(compressed + 8, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 16) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 16)); - w0 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 9, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 17) ) , 3)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 18) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 22)); - w1 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 10, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 19) ) , 9)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 20) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 11, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 21) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 15)); - w1 = _mm256_srli_epi32(tmp,17); - _mm256_storeu_si256(compressed + 12, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 22) ) , 2)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 23) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 21)); - w0 = _mm256_srli_epi32(tmp,11); - _mm256_storeu_si256(compressed + 13, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 24) ) , 8)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 25) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 27)); - w1 = _mm256_srli_epi32(tmp,5); - _mm256_storeu_si256(compressed + 14, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 26) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 14)); - w0 = _mm256_srli_epi32(tmp,18); - _mm256_storeu_si256(compressed + 15, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 27) ) , 1)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 28) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 20)); - w1 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 16, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 29) ) , 7)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 30) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 26)); - w0 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 17, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 31) ) , 13)); - _mm256_storeu_si256(compressed + 18, w0); -} - - -/* we are going to pack 256 20-bit values, touching 20 256-bit words, using 320 bytes */ -static void avxpackblockmask20(const uint32_t * pin, __m256i * compressed) { - /* we are going to touch 20 256-bit words */ - __m256i w0, w1; - const __m256i * in = (const __m256i *) pin; - const __m256i mask = _mm256_set1_epi32(1048575); - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 0) ) ; - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 1) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 20)); - w1 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 2) ) , 8)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 3) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 1, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 4) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 16)); - w1 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 5) ) , 4)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 6) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 3, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 7) ) , 12)); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 8) ) ; - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 9) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 20)); - w0 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 5, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 10) ) , 8)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 11) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 6, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 12) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 16)); - w0 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 7, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 13) ) , 4)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 14) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 8, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 15) ) , 12)); - _mm256_storeu_si256(compressed + 9, w1); - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 16) ) ; - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 17) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 20)); - w1 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 10, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 18) ) , 8)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 19) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 11, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 20) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 16)); - w1 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 12, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 21) ) , 4)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 22) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 13, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 23) ) , 12)); - _mm256_storeu_si256(compressed + 14, w0); - w1 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 24) ) ; - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 25) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 20)); - w0 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 15, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 26) ) , 8)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 27) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 16, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 28) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 16)); - w0 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 17, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 29) ) , 4)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 30) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 18, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 31) ) , 12)); - _mm256_storeu_si256(compressed + 19, w1); -} - - -/* we are going to pack 256 21-bit values, touching 21 256-bit words, using 336 bytes */ -static void avxpackblockmask21(const uint32_t * pin, __m256i * compressed) { - /* we are going to touch 21 256-bit words */ - __m256i w0, w1; - const __m256i * in = (const __m256i *) pin; - const __m256i mask = _mm256_set1_epi32(2097151); - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 0) ) ; - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 1) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 21)); - w1 = _mm256_srli_epi32(tmp,11); - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 2) ) , 10)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 3) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 31)); - w0 = _mm256_srli_epi32(tmp,1); - _mm256_storeu_si256(compressed + 1, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 4) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 20)); - w1 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 5) ) , 9)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 6) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 30)); - w0 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 3, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 7) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 19)); - w1 = _mm256_srli_epi32(tmp,13); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 8) ) , 8)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 9) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 29)); - w0 = _mm256_srli_epi32(tmp,3); - _mm256_storeu_si256(compressed + 5, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 10) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 18)); - w1 = _mm256_srli_epi32(tmp,14); - _mm256_storeu_si256(compressed + 6, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 11) ) , 7)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 12) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 7, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 13) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 17)); - w1 = _mm256_srli_epi32(tmp,15); - _mm256_storeu_si256(compressed + 8, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 14) ) , 6)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 15) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 27)); - w0 = _mm256_srli_epi32(tmp,5); - _mm256_storeu_si256(compressed + 9, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 16) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 16)); - w1 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 10, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 17) ) , 5)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 18) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 26)); - w0 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 11, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 19) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 15)); - w1 = _mm256_srli_epi32(tmp,17); - _mm256_storeu_si256(compressed + 12, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 20) ) , 4)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 21) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 25)); - w0 = _mm256_srli_epi32(tmp,7); - _mm256_storeu_si256(compressed + 13, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 22) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 14)); - w1 = _mm256_srli_epi32(tmp,18); - _mm256_storeu_si256(compressed + 14, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 23) ) , 3)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 24) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 15, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 25) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 13)); - w1 = _mm256_srli_epi32(tmp,19); - _mm256_storeu_si256(compressed + 16, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 26) ) , 2)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 27) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 23)); - w0 = _mm256_srli_epi32(tmp,9); - _mm256_storeu_si256(compressed + 17, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 28) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 12)); - w1 = _mm256_srli_epi32(tmp,20); - _mm256_storeu_si256(compressed + 18, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 29) ) , 1)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 30) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 22)); - w0 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 19, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 31) ) , 11)); - _mm256_storeu_si256(compressed + 20, w0); -} - - -/* we are going to pack 256 22-bit values, touching 22 256-bit words, using 352 bytes */ -static void avxpackblockmask22(const uint32_t * pin, __m256i * compressed) { - /* we are going to touch 22 256-bit words */ - __m256i w0, w1; - const __m256i * in = (const __m256i *) pin; - const __m256i mask = _mm256_set1_epi32(4194303); - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 0) ) ; - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 1) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 22)); - w1 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 0, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 2) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 12)); - w0 = _mm256_srli_epi32(tmp,20); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 3) ) , 2)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 4) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 2, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 5) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 14)); - w0 = _mm256_srli_epi32(tmp,18); - _mm256_storeu_si256(compressed + 3, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 6) ) , 4)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 7) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 26)); - w1 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 4, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 8) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 16)); - w0 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 5, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 9) ) , 6)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 10) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 6, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 11) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 18)); - w0 = _mm256_srli_epi32(tmp,14); - _mm256_storeu_si256(compressed + 7, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 12) ) , 8)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 13) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 30)); - w1 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 8, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 14) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 20)); - w0 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 9, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 15) ) , 10)); - _mm256_storeu_si256(compressed + 10, w0); - w1 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 16) ) ; - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 17) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 22)); - w0 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 11, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 18) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 12)); - w1 = _mm256_srli_epi32(tmp,20); - _mm256_storeu_si256(compressed + 12, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 19) ) , 2)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 20) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 13, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 21) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 14)); - w1 = _mm256_srli_epi32(tmp,18); - _mm256_storeu_si256(compressed + 14, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 22) ) , 4)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 23) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 26)); - w0 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 15, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 24) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 16)); - w1 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 16, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 25) ) , 6)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 26) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 17, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 27) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 18)); - w1 = _mm256_srli_epi32(tmp,14); - _mm256_storeu_si256(compressed + 18, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 28) ) , 8)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 29) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 30)); - w0 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 19, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 30) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 20)); - w1 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 20, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 31) ) , 10)); - _mm256_storeu_si256(compressed + 21, w1); -} - - -/* we are going to pack 256 23-bit values, touching 23 256-bit words, using 368 bytes */ -static void avxpackblockmask23(const uint32_t * pin, __m256i * compressed) { - /* we are going to touch 23 256-bit words */ - __m256i w0, w1; - const __m256i * in = (const __m256i *) pin; - const __m256i mask = _mm256_set1_epi32(8388607); - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 0) ) ; - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 1) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 23)); - w1 = _mm256_srli_epi32(tmp,9); - _mm256_storeu_si256(compressed + 0, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 2) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 14)); - w0 = _mm256_srli_epi32(tmp,18); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 3) ) , 5)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 4) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 2, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 5) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 19)); - w0 = _mm256_srli_epi32(tmp,13); - _mm256_storeu_si256(compressed + 3, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 6) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 10)); - w1 = _mm256_srli_epi32(tmp,22); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 7) ) , 1)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 8) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 5, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 9) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 15)); - w1 = _mm256_srli_epi32(tmp,17); - _mm256_storeu_si256(compressed + 6, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 10) ) , 6)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 11) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 29)); - w0 = _mm256_srli_epi32(tmp,3); - _mm256_storeu_si256(compressed + 7, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 12) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 20)); - w1 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 8, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 13) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 11)); - w0 = _mm256_srli_epi32(tmp,21); - _mm256_storeu_si256(compressed + 9, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 14) ) , 2)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 15) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 25)); - w1 = _mm256_srli_epi32(tmp,7); - _mm256_storeu_si256(compressed + 10, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 16) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 16)); - w0 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 11, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 17) ) , 7)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 18) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 30)); - w1 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 12, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 19) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 21)); - w0 = _mm256_srli_epi32(tmp,11); - _mm256_storeu_si256(compressed + 13, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 20) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 12)); - w1 = _mm256_srli_epi32(tmp,20); - _mm256_storeu_si256(compressed + 14, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 21) ) , 3)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 22) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 26)); - w0 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 15, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 23) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 17)); - w1 = _mm256_srli_epi32(tmp,15); - _mm256_storeu_si256(compressed + 16, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 24) ) , 8)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 25) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 31)); - w0 = _mm256_srli_epi32(tmp,1); - _mm256_storeu_si256(compressed + 17, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 26) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 22)); - w1 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 18, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 27) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 13)); - w0 = _mm256_srli_epi32(tmp,19); - _mm256_storeu_si256(compressed + 19, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 28) ) , 4)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 29) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 27)); - w1 = _mm256_srli_epi32(tmp,5); - _mm256_storeu_si256(compressed + 20, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 30) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 18)); - w0 = _mm256_srli_epi32(tmp,14); - _mm256_storeu_si256(compressed + 21, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 31) ) , 9)); - _mm256_storeu_si256(compressed + 22, w0); -} - - -/* we are going to pack 256 24-bit values, touching 24 256-bit words, using 384 bytes */ -static void avxpackblockmask24(const uint32_t * pin, __m256i * compressed) { - /* we are going to touch 24 256-bit words */ - __m256i w0, w1; - const __m256i * in = (const __m256i *) pin; - const __m256i mask = _mm256_set1_epi32(16777215); - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 0) ) ; - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 1) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 0, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 2) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 16)); - w0 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 3) ) , 8)); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 4) ) ; - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 5) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 3, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 6) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 16)); - w1 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 7) ) , 8)); - _mm256_storeu_si256(compressed + 5, w1); - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 8) ) ; - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 9) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 6, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 10) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 16)); - w0 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 7, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 11) ) , 8)); - _mm256_storeu_si256(compressed + 8, w0); - w1 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 12) ) ; - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 13) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 9, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 14) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 16)); - w1 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 10, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 15) ) , 8)); - _mm256_storeu_si256(compressed + 11, w1); - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 16) ) ; - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 17) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 12, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 18) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 16)); - w0 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 13, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 19) ) , 8)); - _mm256_storeu_si256(compressed + 14, w0); - w1 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 20) ) ; - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 21) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 15, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 22) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 16)); - w1 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 16, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 23) ) , 8)); - _mm256_storeu_si256(compressed + 17, w1); - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 24) ) ; - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 25) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 18, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 26) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 16)); - w0 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 19, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 27) ) , 8)); - _mm256_storeu_si256(compressed + 20, w0); - w1 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 28) ) ; - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 29) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 21, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 30) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 16)); - w1 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 22, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 31) ) , 8)); - _mm256_storeu_si256(compressed + 23, w1); -} - - -/* we are going to pack 256 25-bit values, touching 25 256-bit words, using 400 bytes */ -static void avxpackblockmask25(const uint32_t * pin, __m256i * compressed) { - /* we are going to touch 25 256-bit words */ - __m256i w0, w1; - const __m256i * in = (const __m256i *) pin; - const __m256i mask = _mm256_set1_epi32(33554431); - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 0) ) ; - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 1) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 25)); - w1 = _mm256_srli_epi32(tmp,7); - _mm256_storeu_si256(compressed + 0, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 2) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 18)); - w0 = _mm256_srli_epi32(tmp,14); - _mm256_storeu_si256(compressed + 1, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 3) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 11)); - w1 = _mm256_srli_epi32(tmp,21); - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 4) ) , 4)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 5) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 29)); - w0 = _mm256_srli_epi32(tmp,3); - _mm256_storeu_si256(compressed + 3, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 6) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 22)); - w1 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 4, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 7) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 15)); - w0 = _mm256_srli_epi32(tmp,17); - _mm256_storeu_si256(compressed + 5, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 8) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 8)); - w1 = _mm256_srli_epi32(tmp,24); - _mm256_storeu_si256(compressed + 6, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 9) ) , 1)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 10) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 26)); - w0 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 7, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 11) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 19)); - w1 = _mm256_srli_epi32(tmp,13); - _mm256_storeu_si256(compressed + 8, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 12) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 12)); - w0 = _mm256_srli_epi32(tmp,20); - _mm256_storeu_si256(compressed + 9, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 13) ) , 5)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 14) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 30)); - w1 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 10, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 15) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 23)); - w0 = _mm256_srli_epi32(tmp,9); - _mm256_storeu_si256(compressed + 11, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 16) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 16)); - w1 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 12, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 17) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 9)); - w0 = _mm256_srli_epi32(tmp,23); - _mm256_storeu_si256(compressed + 13, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 18) ) , 2)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 19) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 27)); - w1 = _mm256_srli_epi32(tmp,5); - _mm256_storeu_si256(compressed + 14, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 20) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 20)); - w0 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 15, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 21) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 13)); - w1 = _mm256_srli_epi32(tmp,19); - _mm256_storeu_si256(compressed + 16, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 22) ) , 6)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 23) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 31)); - w0 = _mm256_srli_epi32(tmp,1); - _mm256_storeu_si256(compressed + 17, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 24) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 18, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 25) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 17)); - w0 = _mm256_srli_epi32(tmp,15); - _mm256_storeu_si256(compressed + 19, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 26) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 10)); - w1 = _mm256_srli_epi32(tmp,22); - _mm256_storeu_si256(compressed + 20, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 27) ) , 3)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 28) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 21, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 29) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 21)); - w1 = _mm256_srli_epi32(tmp,11); - _mm256_storeu_si256(compressed + 22, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 30) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 14)); - w0 = _mm256_srli_epi32(tmp,18); - _mm256_storeu_si256(compressed + 23, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 31) ) , 7)); - _mm256_storeu_si256(compressed + 24, w0); -} - - -/* we are going to pack 256 26-bit values, touching 26 256-bit words, using 416 bytes */ -static void avxpackblockmask26(const uint32_t * pin, __m256i * compressed) { - /* we are going to touch 26 256-bit words */ - __m256i w0, w1; - const __m256i * in = (const __m256i *) pin; - const __m256i mask = _mm256_set1_epi32(67108863); - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 0) ) ; - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 1) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 26)); - w1 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 0, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 2) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 20)); - w0 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 1, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 3) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 14)); - w1 = _mm256_srli_epi32(tmp,18); - _mm256_storeu_si256(compressed + 2, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 4) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 8)); - w0 = _mm256_srli_epi32(tmp,24); - _mm256_storeu_si256(compressed + 3, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 5) ) , 2)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 6) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 4, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 7) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 22)); - w0 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 5, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 8) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 16)); - w1 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 6, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 9) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 10)); - w0 = _mm256_srli_epi32(tmp,22); - _mm256_storeu_si256(compressed + 7, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 10) ) , 4)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 11) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 30)); - w1 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 8, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 12) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 9, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 13) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 18)); - w1 = _mm256_srli_epi32(tmp,14); - _mm256_storeu_si256(compressed + 10, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 14) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 12)); - w0 = _mm256_srli_epi32(tmp,20); - _mm256_storeu_si256(compressed + 11, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 15) ) , 6)); - _mm256_storeu_si256(compressed + 12, w0); - w1 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 16) ) ; - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 17) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 26)); - w0 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 13, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 18) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 20)); - w1 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 14, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 19) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 14)); - w0 = _mm256_srli_epi32(tmp,18); - _mm256_storeu_si256(compressed + 15, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 20) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 8)); - w1 = _mm256_srli_epi32(tmp,24); - _mm256_storeu_si256(compressed + 16, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 21) ) , 2)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 22) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 17, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 23) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 22)); - w1 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 18, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 24) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 16)); - w0 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 19, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 25) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 10)); - w1 = _mm256_srli_epi32(tmp,22); - _mm256_storeu_si256(compressed + 20, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 26) ) , 4)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 27) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 30)); - w0 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 21, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 28) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 22, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 29) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 18)); - w0 = _mm256_srli_epi32(tmp,14); - _mm256_storeu_si256(compressed + 23, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 30) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 12)); - w1 = _mm256_srli_epi32(tmp,20); - _mm256_storeu_si256(compressed + 24, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 31) ) , 6)); - _mm256_storeu_si256(compressed + 25, w1); -} - - -/* we are going to pack 256 27-bit values, touching 27 256-bit words, using 432 bytes */ -static void avxpackblockmask27(const uint32_t * pin, __m256i * compressed) { - /* we are going to touch 27 256-bit words */ - __m256i w0, w1; - const __m256i * in = (const __m256i *) pin; - const __m256i mask = _mm256_set1_epi32(134217727); - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 0) ) ; - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 1) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 27)); - w1 = _mm256_srli_epi32(tmp,5); - _mm256_storeu_si256(compressed + 0, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 2) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 22)); - w0 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 1, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 3) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 17)); - w1 = _mm256_srli_epi32(tmp,15); - _mm256_storeu_si256(compressed + 2, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 4) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 12)); - w0 = _mm256_srli_epi32(tmp,20); - _mm256_storeu_si256(compressed + 3, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 5) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 7)); - w1 = _mm256_srli_epi32(tmp,25); - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 6) ) , 2)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 7) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 29)); - w0 = _mm256_srli_epi32(tmp,3); - _mm256_storeu_si256(compressed + 5, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 8) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 6, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 9) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 19)); - w0 = _mm256_srli_epi32(tmp,13); - _mm256_storeu_si256(compressed + 7, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 10) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 14)); - w1 = _mm256_srli_epi32(tmp,18); - _mm256_storeu_si256(compressed + 8, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 11) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 9)); - w0 = _mm256_srli_epi32(tmp,23); - _mm256_storeu_si256(compressed + 9, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 12) ) , 4)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 13) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 31)); - w1 = _mm256_srli_epi32(tmp,1); - _mm256_storeu_si256(compressed + 10, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 14) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 26)); - w0 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 11, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 15) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 21)); - w1 = _mm256_srli_epi32(tmp,11); - _mm256_storeu_si256(compressed + 12, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 16) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 16)); - w0 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 13, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 17) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 11)); - w1 = _mm256_srli_epi32(tmp,21); - _mm256_storeu_si256(compressed + 14, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 18) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 6)); - w0 = _mm256_srli_epi32(tmp,26); - _mm256_storeu_si256(compressed + 15, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 19) ) , 1)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 20) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 16, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 21) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 23)); - w0 = _mm256_srli_epi32(tmp,9); - _mm256_storeu_si256(compressed + 17, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 22) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 18)); - w1 = _mm256_srli_epi32(tmp,14); - _mm256_storeu_si256(compressed + 18, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 23) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 13)); - w0 = _mm256_srli_epi32(tmp,19); - _mm256_storeu_si256(compressed + 19, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 24) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 8)); - w1 = _mm256_srli_epi32(tmp,24); - _mm256_storeu_si256(compressed + 20, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 25) ) , 3)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 26) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 30)); - w0 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 21, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 27) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 25)); - w1 = _mm256_srli_epi32(tmp,7); - _mm256_storeu_si256(compressed + 22, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 28) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 20)); - w0 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 23, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 29) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 15)); - w1 = _mm256_srli_epi32(tmp,17); - _mm256_storeu_si256(compressed + 24, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 30) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 10)); - w0 = _mm256_srli_epi32(tmp,22); - _mm256_storeu_si256(compressed + 25, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 31) ) , 5)); - _mm256_storeu_si256(compressed + 26, w0); -} - - -/* we are going to pack 256 28-bit values, touching 28 256-bit words, using 448 bytes */ -static void avxpackblockmask28(const uint32_t * pin, __m256i * compressed) { - /* we are going to touch 28 256-bit words */ - __m256i w0, w1; - const __m256i * in = (const __m256i *) pin; - const __m256i mask = _mm256_set1_epi32(268435455); - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 0) ) ; - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 1) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 0, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 2) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 1, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 3) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 20)); - w1 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 2, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 4) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 16)); - w0 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 3, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 5) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 12)); - w1 = _mm256_srli_epi32(tmp,20); - _mm256_storeu_si256(compressed + 4, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 6) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 8)); - w0 = _mm256_srli_epi32(tmp,24); - _mm256_storeu_si256(compressed + 5, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 7) ) , 4)); - _mm256_storeu_si256(compressed + 6, w0); - w1 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 8) ) ; - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 9) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 7, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 10) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 8, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 11) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 20)); - w0 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 9, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 12) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 16)); - w1 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 10, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 13) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 12)); - w0 = _mm256_srli_epi32(tmp,20); - _mm256_storeu_si256(compressed + 11, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 14) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 8)); - w1 = _mm256_srli_epi32(tmp,24); - _mm256_storeu_si256(compressed + 12, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 15) ) , 4)); - _mm256_storeu_si256(compressed + 13, w1); - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 16) ) ; - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 17) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 14, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 18) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 15, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 19) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 20)); - w1 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 16, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 20) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 16)); - w0 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 17, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 21) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 12)); - w1 = _mm256_srli_epi32(tmp,20); - _mm256_storeu_si256(compressed + 18, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 22) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 8)); - w0 = _mm256_srli_epi32(tmp,24); - _mm256_storeu_si256(compressed + 19, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 23) ) , 4)); - _mm256_storeu_si256(compressed + 20, w0); - w1 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 24) ) ; - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 25) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 21, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 26) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 22, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 27) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 20)); - w0 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 23, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 28) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 16)); - w1 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 24, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 29) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 12)); - w0 = _mm256_srli_epi32(tmp,20); - _mm256_storeu_si256(compressed + 25, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 30) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 8)); - w1 = _mm256_srli_epi32(tmp,24); - _mm256_storeu_si256(compressed + 26, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 31) ) , 4)); - _mm256_storeu_si256(compressed + 27, w1); -} - - -/* we are going to pack 256 29-bit values, touching 29 256-bit words, using 464 bytes */ -static void avxpackblockmask29(const uint32_t * pin, __m256i * compressed) { - /* we are going to touch 29 256-bit words */ - __m256i w0, w1; - const __m256i * in = (const __m256i *) pin; - const __m256i mask = _mm256_set1_epi32(536870911); - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 0) ) ; - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 1) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 29)); - w1 = _mm256_srli_epi32(tmp,3); - _mm256_storeu_si256(compressed + 0, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 2) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 26)); - w0 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 1, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 3) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 23)); - w1 = _mm256_srli_epi32(tmp,9); - _mm256_storeu_si256(compressed + 2, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 4) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 20)); - w0 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 3, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 5) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 17)); - w1 = _mm256_srli_epi32(tmp,15); - _mm256_storeu_si256(compressed + 4, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 6) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 14)); - w0 = _mm256_srli_epi32(tmp,18); - _mm256_storeu_si256(compressed + 5, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 7) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 11)); - w1 = _mm256_srli_epi32(tmp,21); - _mm256_storeu_si256(compressed + 6, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 8) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 8)); - w0 = _mm256_srli_epi32(tmp,24); - _mm256_storeu_si256(compressed + 7, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 9) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 5)); - w1 = _mm256_srli_epi32(tmp,27); - _mm256_storeu_si256(compressed + 8, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 10) ) , 2)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 11) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 31)); - w0 = _mm256_srli_epi32(tmp,1); - _mm256_storeu_si256(compressed + 9, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 12) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 10, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 13) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 25)); - w0 = _mm256_srli_epi32(tmp,7); - _mm256_storeu_si256(compressed + 11, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 14) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 22)); - w1 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 12, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 15) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 19)); - w0 = _mm256_srli_epi32(tmp,13); - _mm256_storeu_si256(compressed + 13, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 16) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 16)); - w1 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 14, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 17) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 13)); - w0 = _mm256_srli_epi32(tmp,19); - _mm256_storeu_si256(compressed + 15, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 18) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 10)); - w1 = _mm256_srli_epi32(tmp,22); - _mm256_storeu_si256(compressed + 16, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 19) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 7)); - w0 = _mm256_srli_epi32(tmp,25); - _mm256_storeu_si256(compressed + 17, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 20) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 4)); - w1 = _mm256_srli_epi32(tmp,28); - _mm256_storeu_si256(compressed + 18, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 21) ) , 1)); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 22) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 30)); - w0 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 19, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 23) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 27)); - w1 = _mm256_srli_epi32(tmp,5); - _mm256_storeu_si256(compressed + 20, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 24) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 21, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 25) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 21)); - w1 = _mm256_srli_epi32(tmp,11); - _mm256_storeu_si256(compressed + 22, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 26) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 18)); - w0 = _mm256_srli_epi32(tmp,14); - _mm256_storeu_si256(compressed + 23, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 27) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 15)); - w1 = _mm256_srli_epi32(tmp,17); - _mm256_storeu_si256(compressed + 24, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 28) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 12)); - w0 = _mm256_srli_epi32(tmp,20); - _mm256_storeu_si256(compressed + 25, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 29) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 9)); - w1 = _mm256_srli_epi32(tmp,23); - _mm256_storeu_si256(compressed + 26, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 30) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 6)); - w0 = _mm256_srli_epi32(tmp,26); - _mm256_storeu_si256(compressed + 27, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 31) ) , 3)); - _mm256_storeu_si256(compressed + 28, w0); -} - - -/* we are going to pack 256 30-bit values, touching 30 256-bit words, using 480 bytes */ -static void avxpackblockmask30(const uint32_t * pin, __m256i * compressed) { - /* we are going to touch 30 256-bit words */ - __m256i w0, w1; - const __m256i * in = (const __m256i *) pin; - const __m256i mask = _mm256_set1_epi32(1073741823); - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 0) ) ; - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 1) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 30)); - w1 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 0, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 2) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 1, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 3) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 26)); - w1 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 2, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 4) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 3, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 5) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 22)); - w1 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 4, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 6) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 20)); - w0 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 5, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 7) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 18)); - w1 = _mm256_srli_epi32(tmp,14); - _mm256_storeu_si256(compressed + 6, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 8) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 16)); - w0 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 7, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 9) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 14)); - w1 = _mm256_srli_epi32(tmp,18); - _mm256_storeu_si256(compressed + 8, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 10) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 12)); - w0 = _mm256_srli_epi32(tmp,20); - _mm256_storeu_si256(compressed + 9, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 11) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 10)); - w1 = _mm256_srli_epi32(tmp,22); - _mm256_storeu_si256(compressed + 10, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 12) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 8)); - w0 = _mm256_srli_epi32(tmp,24); - _mm256_storeu_si256(compressed + 11, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 13) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 6)); - w1 = _mm256_srli_epi32(tmp,26); - _mm256_storeu_si256(compressed + 12, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 14) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 4)); - w0 = _mm256_srli_epi32(tmp,28); - _mm256_storeu_si256(compressed + 13, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 15) ) , 2)); - _mm256_storeu_si256(compressed + 14, w0); - w1 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 16) ) ; - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 17) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 30)); - w0 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 15, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 18) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 28)); - w1 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 16, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 19) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 26)); - w0 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 17, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 20) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 24)); - w1 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 18, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 21) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 22)); - w0 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 19, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 22) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 20)); - w1 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 20, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 23) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 18)); - w0 = _mm256_srli_epi32(tmp,14); - _mm256_storeu_si256(compressed + 21, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 24) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 16)); - w1 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 22, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 25) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 14)); - w0 = _mm256_srli_epi32(tmp,18); - _mm256_storeu_si256(compressed + 23, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 26) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 12)); - w1 = _mm256_srli_epi32(tmp,20); - _mm256_storeu_si256(compressed + 24, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 27) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 10)); - w0 = _mm256_srli_epi32(tmp,22); - _mm256_storeu_si256(compressed + 25, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 28) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 8)); - w1 = _mm256_srli_epi32(tmp,24); - _mm256_storeu_si256(compressed + 26, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 29) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 6)); - w0 = _mm256_srli_epi32(tmp,26); - _mm256_storeu_si256(compressed + 27, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 30) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 4)); - w1 = _mm256_srli_epi32(tmp,28); - _mm256_storeu_si256(compressed + 28, w0); - w1 = _mm256_or_si256(w1,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 31) ) , 2)); - _mm256_storeu_si256(compressed + 29, w1); -} - - -/* we are going to pack 256 31-bit values, touching 31 256-bit words, using 496 bytes */ -static void avxpackblockmask31(const uint32_t * pin, __m256i * compressed) { - /* we are going to touch 31 256-bit words */ - __m256i w0, w1; - const __m256i * in = (const __m256i *) pin; - const __m256i mask = _mm256_set1_epi32(2147483647); - __m256i tmp; /* used to store inputs at word boundary */ - w0 = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 0) ) ; - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 1) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 31)); - w1 = _mm256_srli_epi32(tmp,1); - _mm256_storeu_si256(compressed + 0, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 2) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 30)); - w0 = _mm256_srli_epi32(tmp,2); - _mm256_storeu_si256(compressed + 1, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 3) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 29)); - w1 = _mm256_srli_epi32(tmp,3); - _mm256_storeu_si256(compressed + 2, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 4) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 28)); - w0 = _mm256_srli_epi32(tmp,4); - _mm256_storeu_si256(compressed + 3, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 5) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 27)); - w1 = _mm256_srli_epi32(tmp,5); - _mm256_storeu_si256(compressed + 4, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 6) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 26)); - w0 = _mm256_srli_epi32(tmp,6); - _mm256_storeu_si256(compressed + 5, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 7) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 25)); - w1 = _mm256_srli_epi32(tmp,7); - _mm256_storeu_si256(compressed + 6, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 8) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 24)); - w0 = _mm256_srli_epi32(tmp,8); - _mm256_storeu_si256(compressed + 7, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 9) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 23)); - w1 = _mm256_srli_epi32(tmp,9); - _mm256_storeu_si256(compressed + 8, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 10) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 22)); - w0 = _mm256_srli_epi32(tmp,10); - _mm256_storeu_si256(compressed + 9, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 11) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 21)); - w1 = _mm256_srli_epi32(tmp,11); - _mm256_storeu_si256(compressed + 10, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 12) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 20)); - w0 = _mm256_srli_epi32(tmp,12); - _mm256_storeu_si256(compressed + 11, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 13) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 19)); - w1 = _mm256_srli_epi32(tmp,13); - _mm256_storeu_si256(compressed + 12, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 14) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 18)); - w0 = _mm256_srli_epi32(tmp,14); - _mm256_storeu_si256(compressed + 13, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 15) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 17)); - w1 = _mm256_srli_epi32(tmp,15); - _mm256_storeu_si256(compressed + 14, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 16) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 16)); - w0 = _mm256_srli_epi32(tmp,16); - _mm256_storeu_si256(compressed + 15, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 17) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 15)); - w1 = _mm256_srli_epi32(tmp,17); - _mm256_storeu_si256(compressed + 16, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 18) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 14)); - w0 = _mm256_srli_epi32(tmp,18); - _mm256_storeu_si256(compressed + 17, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 19) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 13)); - w1 = _mm256_srli_epi32(tmp,19); - _mm256_storeu_si256(compressed + 18, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 20) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 12)); - w0 = _mm256_srli_epi32(tmp,20); - _mm256_storeu_si256(compressed + 19, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 21) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 11)); - w1 = _mm256_srli_epi32(tmp,21); - _mm256_storeu_si256(compressed + 20, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 22) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 10)); - w0 = _mm256_srli_epi32(tmp,22); - _mm256_storeu_si256(compressed + 21, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 23) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 9)); - w1 = _mm256_srli_epi32(tmp,23); - _mm256_storeu_si256(compressed + 22, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 24) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 8)); - w0 = _mm256_srli_epi32(tmp,24); - _mm256_storeu_si256(compressed + 23, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 25) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 7)); - w1 = _mm256_srli_epi32(tmp,25); - _mm256_storeu_si256(compressed + 24, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 26) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 6)); - w0 = _mm256_srli_epi32(tmp,26); - _mm256_storeu_si256(compressed + 25, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 27) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 5)); - w1 = _mm256_srli_epi32(tmp,27); - _mm256_storeu_si256(compressed + 26, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 28) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 4)); - w0 = _mm256_srli_epi32(tmp,28); - _mm256_storeu_si256(compressed + 27, w1); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 29) ) ; - w0 = _mm256_or_si256(w0,_mm256_slli_epi32(tmp , 3)); - w1 = _mm256_srli_epi32(tmp,29); - _mm256_storeu_si256(compressed + 28, w0); - tmp = _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 30) ) ; - w1 = _mm256_or_si256(w1,_mm256_slli_epi32(tmp , 2)); - w0 = _mm256_srli_epi32(tmp,30); - _mm256_storeu_si256(compressed + 29, w1); - w0 = _mm256_or_si256(w0,_mm256_slli_epi32( _mm256_and_si256 ( mask, _mm256_lddqu_si256 (in + 31) ) , 1)); - _mm256_storeu_si256(compressed + 30, w0); -} - - -/* we are going to pack 256 32-bit values, touching 32 256-bit words, using 512 bytes */ -static void avxpackblockmask32(const uint32_t * pin, __m256i * compressed) { - /* we are going to touch 32 256-bit words */ - __m256i w0, w1; - const __m256i * in = (const __m256i *) pin; - w0 = _mm256_lddqu_si256 (in + 0) ; - _mm256_storeu_si256(compressed + 0, w0); - w1 = _mm256_lddqu_si256 (in + 1) ; - _mm256_storeu_si256(compressed + 1, w1); - w0 = _mm256_lddqu_si256 (in + 2) ; - _mm256_storeu_si256(compressed + 2, w0); - w1 = _mm256_lddqu_si256 (in + 3) ; - _mm256_storeu_si256(compressed + 3, w1); - w0 = _mm256_lddqu_si256 (in + 4) ; - _mm256_storeu_si256(compressed + 4, w0); - w1 = _mm256_lddqu_si256 (in + 5) ; - _mm256_storeu_si256(compressed + 5, w1); - w0 = _mm256_lddqu_si256 (in + 6) ; - _mm256_storeu_si256(compressed + 6, w0); - w1 = _mm256_lddqu_si256 (in + 7) ; - _mm256_storeu_si256(compressed + 7, w1); - w0 = _mm256_lddqu_si256 (in + 8) ; - _mm256_storeu_si256(compressed + 8, w0); - w1 = _mm256_lddqu_si256 (in + 9) ; - _mm256_storeu_si256(compressed + 9, w1); - w0 = _mm256_lddqu_si256 (in + 10) ; - _mm256_storeu_si256(compressed + 10, w0); - w1 = _mm256_lddqu_si256 (in + 11) ; - _mm256_storeu_si256(compressed + 11, w1); - w0 = _mm256_lddqu_si256 (in + 12) ; - _mm256_storeu_si256(compressed + 12, w0); - w1 = _mm256_lddqu_si256 (in + 13) ; - _mm256_storeu_si256(compressed + 13, w1); - w0 = _mm256_lddqu_si256 (in + 14) ; - _mm256_storeu_si256(compressed + 14, w0); - w1 = _mm256_lddqu_si256 (in + 15) ; - _mm256_storeu_si256(compressed + 15, w1); - w0 = _mm256_lddqu_si256 (in + 16) ; - _mm256_storeu_si256(compressed + 16, w0); - w1 = _mm256_lddqu_si256 (in + 17) ; - _mm256_storeu_si256(compressed + 17, w1); - w0 = _mm256_lddqu_si256 (in + 18) ; - _mm256_storeu_si256(compressed + 18, w0); - w1 = _mm256_lddqu_si256 (in + 19) ; - _mm256_storeu_si256(compressed + 19, w1); - w0 = _mm256_lddqu_si256 (in + 20) ; - _mm256_storeu_si256(compressed + 20, w0); - w1 = _mm256_lddqu_si256 (in + 21) ; - _mm256_storeu_si256(compressed + 21, w1); - w0 = _mm256_lddqu_si256 (in + 22) ; - _mm256_storeu_si256(compressed + 22, w0); - w1 = _mm256_lddqu_si256 (in + 23) ; - _mm256_storeu_si256(compressed + 23, w1); - w0 = _mm256_lddqu_si256 (in + 24) ; - _mm256_storeu_si256(compressed + 24, w0); - w1 = _mm256_lddqu_si256 (in + 25) ; - _mm256_storeu_si256(compressed + 25, w1); - w0 = _mm256_lddqu_si256 (in + 26) ; - _mm256_storeu_si256(compressed + 26, w0); - w1 = _mm256_lddqu_si256 (in + 27) ; - _mm256_storeu_si256(compressed + 27, w1); - w0 = _mm256_lddqu_si256 (in + 28) ; - _mm256_storeu_si256(compressed + 28, w0); - w1 = _mm256_lddqu_si256 (in + 29) ; - _mm256_storeu_si256(compressed + 29, w1); - w0 = _mm256_lddqu_si256 (in + 30) ; - _mm256_storeu_si256(compressed + 30, w0); - w1 = _mm256_lddqu_si256 (in + 31) ; - _mm256_storeu_si256(compressed + 31, w1); -} - -static void avxunpackblock0(const __m256i * compressed, uint32_t * pout) { - (void) compressed; - memset(pout,0,256); -} - - -/* we packed 256 1-bit values, touching 1 256-bit words, using 16 bytes */ -static void avxunpackblock1(const __m256i * compressed, uint32_t * pout) { - /* we are going to access 1 256-bit word */ - __m256i w0; - __m256i * out = (__m256i *) pout; - const __m256i mask = _mm256_set1_epi32(1); - w0 = _mm256_lddqu_si256 (compressed); - _mm256_storeu_si256(out + 0, _mm256_and_si256 ( mask, w0 ) ); - _mm256_storeu_si256(out + 1, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 1) ) ); - _mm256_storeu_si256(out + 2, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 2) ) ); - _mm256_storeu_si256(out + 3, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 3) ) ); - _mm256_storeu_si256(out + 4, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 4) ) ); - _mm256_storeu_si256(out + 5, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 5) ) ); - _mm256_storeu_si256(out + 6, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 6) ) ); - _mm256_storeu_si256(out + 7, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 7) ) ); - _mm256_storeu_si256(out + 8, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 8) ) ); - _mm256_storeu_si256(out + 9, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 9) ) ); - _mm256_storeu_si256(out + 10, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 10) ) ); - _mm256_storeu_si256(out + 11, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 11) ) ); - _mm256_storeu_si256(out + 12, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 12) ) ); - _mm256_storeu_si256(out + 13, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 13) ) ); - _mm256_storeu_si256(out + 14, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 14) ) ); - _mm256_storeu_si256(out + 15, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 15) ) ); - _mm256_storeu_si256(out + 16, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 16) ) ); - _mm256_storeu_si256(out + 17, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 17) ) ); - _mm256_storeu_si256(out + 18, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 18) ) ); - _mm256_storeu_si256(out + 19, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 19) ) ); - _mm256_storeu_si256(out + 20, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 20) ) ); - _mm256_storeu_si256(out + 21, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 21) ) ); - _mm256_storeu_si256(out + 22, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 22) ) ); - _mm256_storeu_si256(out + 23, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 23) ) ); - _mm256_storeu_si256(out + 24, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 24) ) ); - _mm256_storeu_si256(out + 25, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 25) ) ); - _mm256_storeu_si256(out + 26, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 26) ) ); - _mm256_storeu_si256(out + 27, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 27) ) ); - _mm256_storeu_si256(out + 28, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 28) ) ); - _mm256_storeu_si256(out + 29, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 29) ) ); - _mm256_storeu_si256(out + 30, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 30) ) ); - _mm256_storeu_si256(out + 31, _mm256_srli_epi32( w0 , 31) ); -} - - -/* we packed 256 2-bit values, touching 2 256-bit words, using 32 bytes */ -static void avxunpackblock2(const __m256i * compressed, uint32_t * pout) { - /* we are going to access 2 256-bit words */ - __m256i w0, w1; - __m256i * out = (__m256i *) pout; - const __m256i mask = _mm256_set1_epi32(3); - w0 = _mm256_lddqu_si256 (compressed); - _mm256_storeu_si256(out + 0, _mm256_and_si256 ( mask, w0 ) ); - _mm256_storeu_si256(out + 1, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 2) ) ); - _mm256_storeu_si256(out + 2, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 4) ) ); - _mm256_storeu_si256(out + 3, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 6) ) ); - _mm256_storeu_si256(out + 4, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 8) ) ); - _mm256_storeu_si256(out + 5, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 10) ) ); - _mm256_storeu_si256(out + 6, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 12) ) ); - _mm256_storeu_si256(out + 7, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 14) ) ); - _mm256_storeu_si256(out + 8, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 16) ) ); - _mm256_storeu_si256(out + 9, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 18) ) ); - _mm256_storeu_si256(out + 10, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 20) ) ); - _mm256_storeu_si256(out + 11, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 22) ) ); - _mm256_storeu_si256(out + 12, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 24) ) ); - _mm256_storeu_si256(out + 13, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 26) ) ); - _mm256_storeu_si256(out + 14, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 28) ) ); - _mm256_storeu_si256(out + 15, _mm256_srli_epi32( w0 , 30) ); - w1 = _mm256_lddqu_si256 (compressed + 1); - _mm256_storeu_si256(out + 16, _mm256_and_si256 ( mask, w1 ) ); - _mm256_storeu_si256(out + 17, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 2) ) ); - _mm256_storeu_si256(out + 18, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 4) ) ); - _mm256_storeu_si256(out + 19, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 6) ) ); - _mm256_storeu_si256(out + 20, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 8) ) ); - _mm256_storeu_si256(out + 21, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 10) ) ); - _mm256_storeu_si256(out + 22, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 12) ) ); - _mm256_storeu_si256(out + 23, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 14) ) ); - _mm256_storeu_si256(out + 24, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 16) ) ); - _mm256_storeu_si256(out + 25, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 18) ) ); - _mm256_storeu_si256(out + 26, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 20) ) ); - _mm256_storeu_si256(out + 27, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 22) ) ); - _mm256_storeu_si256(out + 28, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 24) ) ); - _mm256_storeu_si256(out + 29, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 26) ) ); - _mm256_storeu_si256(out + 30, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 28) ) ); - _mm256_storeu_si256(out + 31, _mm256_srli_epi32( w1 , 30) ); -} - - -/* we packed 256 3-bit values, touching 3 256-bit words, using 48 bytes */ -static void avxunpackblock3(const __m256i * compressed, uint32_t * pout) { - /* we are going to access 3 256-bit words */ - __m256i w0, w1; - __m256i * out = (__m256i *) pout; - const __m256i mask = _mm256_set1_epi32(7); - w0 = _mm256_lddqu_si256 (compressed); - _mm256_storeu_si256(out + 0, _mm256_and_si256 ( mask, w0 ) ); - _mm256_storeu_si256(out + 1, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 3) ) ); - _mm256_storeu_si256(out + 2, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 6) ) ); - _mm256_storeu_si256(out + 3, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 9) ) ); - _mm256_storeu_si256(out + 4, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 12) ) ); - _mm256_storeu_si256(out + 5, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 15) ) ); - _mm256_storeu_si256(out + 6, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 18) ) ); - _mm256_storeu_si256(out + 7, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 21) ) ); - _mm256_storeu_si256(out + 8, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 24) ) ); - _mm256_storeu_si256(out + 9, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 27) ) ); - w1 = _mm256_lddqu_si256 (compressed + 1); - _mm256_storeu_si256(out + 10, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 30) ,_mm256_slli_epi32( w1 , 2 ) ) ) ); - _mm256_storeu_si256(out + 11, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 1) ) ); - _mm256_storeu_si256(out + 12, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 4) ) ); - _mm256_storeu_si256(out + 13, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 7) ) ); - _mm256_storeu_si256(out + 14, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 10) ) ); - _mm256_storeu_si256(out + 15, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 13) ) ); - _mm256_storeu_si256(out + 16, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 16) ) ); - _mm256_storeu_si256(out + 17, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 19) ) ); - _mm256_storeu_si256(out + 18, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 22) ) ); - _mm256_storeu_si256(out + 19, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 25) ) ); - _mm256_storeu_si256(out + 20, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 28) ) ); - w0 = _mm256_lddqu_si256 (compressed + 2); - _mm256_storeu_si256(out + 21, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 31) ,_mm256_slli_epi32( w0 , 1 ) ) ) ); - _mm256_storeu_si256(out + 22, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 2) ) ); - _mm256_storeu_si256(out + 23, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 5) ) ); - _mm256_storeu_si256(out + 24, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 8) ) ); - _mm256_storeu_si256(out + 25, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 11) ) ); - _mm256_storeu_si256(out + 26, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 14) ) ); - _mm256_storeu_si256(out + 27, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 17) ) ); - _mm256_storeu_si256(out + 28, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 20) ) ); - _mm256_storeu_si256(out + 29, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 23) ) ); - _mm256_storeu_si256(out + 30, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 26) ) ); - _mm256_storeu_si256(out + 31, _mm256_srli_epi32( w0 , 29) ); -} - - -/* we packed 256 4-bit values, touching 4 256-bit words, using 64 bytes */ -static void avxunpackblock4(const __m256i * compressed, uint32_t * pout) { - /* we are going to access 4 256-bit words */ - __m256i w0, w1; - __m256i * out = (__m256i *) pout; - const __m256i mask = _mm256_set1_epi32(15); - w0 = _mm256_lddqu_si256 (compressed); - _mm256_storeu_si256(out + 0, _mm256_and_si256 ( mask, w0 ) ); - _mm256_storeu_si256(out + 1, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 4) ) ); - _mm256_storeu_si256(out + 2, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 8) ) ); - _mm256_storeu_si256(out + 3, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 12) ) ); - _mm256_storeu_si256(out + 4, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 16) ) ); - _mm256_storeu_si256(out + 5, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 20) ) ); - _mm256_storeu_si256(out + 6, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 24) ) ); - _mm256_storeu_si256(out + 7, _mm256_srli_epi32( w0 , 28) ); - w1 = _mm256_lddqu_si256 (compressed + 1); - _mm256_storeu_si256(out + 8, _mm256_and_si256 ( mask, w1 ) ); - _mm256_storeu_si256(out + 9, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 4) ) ); - _mm256_storeu_si256(out + 10, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 8) ) ); - _mm256_storeu_si256(out + 11, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 12) ) ); - _mm256_storeu_si256(out + 12, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 16) ) ); - _mm256_storeu_si256(out + 13, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 20) ) ); - _mm256_storeu_si256(out + 14, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 24) ) ); - _mm256_storeu_si256(out + 15, _mm256_srli_epi32( w1 , 28) ); - w0 = _mm256_lddqu_si256 (compressed + 2); - _mm256_storeu_si256(out + 16, _mm256_and_si256 ( mask, w0 ) ); - _mm256_storeu_si256(out + 17, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 4) ) ); - _mm256_storeu_si256(out + 18, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 8) ) ); - _mm256_storeu_si256(out + 19, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 12) ) ); - _mm256_storeu_si256(out + 20, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 16) ) ); - _mm256_storeu_si256(out + 21, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 20) ) ); - _mm256_storeu_si256(out + 22, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 24) ) ); - _mm256_storeu_si256(out + 23, _mm256_srli_epi32( w0 , 28) ); - w1 = _mm256_lddqu_si256 (compressed + 3); - _mm256_storeu_si256(out + 24, _mm256_and_si256 ( mask, w1 ) ); - _mm256_storeu_si256(out + 25, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 4) ) ); - _mm256_storeu_si256(out + 26, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 8) ) ); - _mm256_storeu_si256(out + 27, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 12) ) ); - _mm256_storeu_si256(out + 28, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 16) ) ); - _mm256_storeu_si256(out + 29, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 20) ) ); - _mm256_storeu_si256(out + 30, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 24) ) ); - _mm256_storeu_si256(out + 31, _mm256_srli_epi32( w1 , 28) ); -} - - -/* we packed 256 5-bit values, touching 5 256-bit words, using 80 bytes */ -static void avxunpackblock5(const __m256i * compressed, uint32_t * pout) { - /* we are going to access 5 256-bit words */ - __m256i w0, w1; - __m256i * out = (__m256i *) pout; - const __m256i mask = _mm256_set1_epi32(31); - w0 = _mm256_lddqu_si256 (compressed); - _mm256_storeu_si256(out + 0, _mm256_and_si256 ( mask, w0 ) ); - _mm256_storeu_si256(out + 1, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 5) ) ); - _mm256_storeu_si256(out + 2, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 10) ) ); - _mm256_storeu_si256(out + 3, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 15) ) ); - _mm256_storeu_si256(out + 4, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 20) ) ); - _mm256_storeu_si256(out + 5, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 25) ) ); - w1 = _mm256_lddqu_si256 (compressed + 1); - _mm256_storeu_si256(out + 6, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 30) ,_mm256_slli_epi32( w1 , 2 ) ) ) ); - _mm256_storeu_si256(out + 7, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 3) ) ); - _mm256_storeu_si256(out + 8, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 8) ) ); - _mm256_storeu_si256(out + 9, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 13) ) ); - _mm256_storeu_si256(out + 10, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 18) ) ); - _mm256_storeu_si256(out + 11, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 23) ) ); - w0 = _mm256_lddqu_si256 (compressed + 2); - _mm256_storeu_si256(out + 12, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 28) ,_mm256_slli_epi32( w0 , 4 ) ) ) ); - _mm256_storeu_si256(out + 13, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 1) ) ); - _mm256_storeu_si256(out + 14, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 6) ) ); - _mm256_storeu_si256(out + 15, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 11) ) ); - _mm256_storeu_si256(out + 16, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 16) ) ); - _mm256_storeu_si256(out + 17, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 21) ) ); - _mm256_storeu_si256(out + 18, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 26) ) ); - w1 = _mm256_lddqu_si256 (compressed + 3); - _mm256_storeu_si256(out + 19, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 31) ,_mm256_slli_epi32( w1 , 1 ) ) ) ); - _mm256_storeu_si256(out + 20, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 4) ) ); - _mm256_storeu_si256(out + 21, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 9) ) ); - _mm256_storeu_si256(out + 22, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 14) ) ); - _mm256_storeu_si256(out + 23, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 19) ) ); - _mm256_storeu_si256(out + 24, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 24) ) ); - w0 = _mm256_lddqu_si256 (compressed + 4); - _mm256_storeu_si256(out + 25, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 29) ,_mm256_slli_epi32( w0 , 3 ) ) ) ); - _mm256_storeu_si256(out + 26, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 2) ) ); - _mm256_storeu_si256(out + 27, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 7) ) ); - _mm256_storeu_si256(out + 28, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 12) ) ); - _mm256_storeu_si256(out + 29, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 17) ) ); - _mm256_storeu_si256(out + 30, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 22) ) ); - _mm256_storeu_si256(out + 31, _mm256_srli_epi32( w0 , 27) ); -} - - -/* we packed 256 6-bit values, touching 6 256-bit words, using 96 bytes */ -static void avxunpackblock6(const __m256i * compressed, uint32_t * pout) { - /* we are going to access 6 256-bit words */ - __m256i w0, w1; - __m256i * out = (__m256i *) pout; - const __m256i mask = _mm256_set1_epi32(63); - w0 = _mm256_lddqu_si256 (compressed); - _mm256_storeu_si256(out + 0, _mm256_and_si256 ( mask, w0 ) ); - _mm256_storeu_si256(out + 1, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 6) ) ); - _mm256_storeu_si256(out + 2, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 12) ) ); - _mm256_storeu_si256(out + 3, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 18) ) ); - _mm256_storeu_si256(out + 4, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 24) ) ); - w1 = _mm256_lddqu_si256 (compressed + 1); - _mm256_storeu_si256(out + 5, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 30) ,_mm256_slli_epi32( w1 , 2 ) ) ) ); - _mm256_storeu_si256(out + 6, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 4) ) ); - _mm256_storeu_si256(out + 7, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 10) ) ); - _mm256_storeu_si256(out + 8, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 16) ) ); - _mm256_storeu_si256(out + 9, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 22) ) ); - w0 = _mm256_lddqu_si256 (compressed + 2); - _mm256_storeu_si256(out + 10, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 28) ,_mm256_slli_epi32( w0 , 4 ) ) ) ); - _mm256_storeu_si256(out + 11, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 2) ) ); - _mm256_storeu_si256(out + 12, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 8) ) ); - _mm256_storeu_si256(out + 13, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 14) ) ); - _mm256_storeu_si256(out + 14, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 20) ) ); - _mm256_storeu_si256(out + 15, _mm256_srli_epi32( w0 , 26) ); - w1 = _mm256_lddqu_si256 (compressed + 3); - _mm256_storeu_si256(out + 16, _mm256_and_si256 ( mask, w1 ) ); - _mm256_storeu_si256(out + 17, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 6) ) ); - _mm256_storeu_si256(out + 18, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 12) ) ); - _mm256_storeu_si256(out + 19, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 18) ) ); - _mm256_storeu_si256(out + 20, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 24) ) ); - w0 = _mm256_lddqu_si256 (compressed + 4); - _mm256_storeu_si256(out + 21, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 30) ,_mm256_slli_epi32( w0 , 2 ) ) ) ); - _mm256_storeu_si256(out + 22, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 4) ) ); - _mm256_storeu_si256(out + 23, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 10) ) ); - _mm256_storeu_si256(out + 24, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 16) ) ); - _mm256_storeu_si256(out + 25, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 22) ) ); - w1 = _mm256_lddqu_si256 (compressed + 5); - _mm256_storeu_si256(out + 26, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 28) ,_mm256_slli_epi32( w1 , 4 ) ) ) ); - _mm256_storeu_si256(out + 27, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 2) ) ); - _mm256_storeu_si256(out + 28, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 8) ) ); - _mm256_storeu_si256(out + 29, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 14) ) ); - _mm256_storeu_si256(out + 30, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 20) ) ); - _mm256_storeu_si256(out + 31, _mm256_srli_epi32( w1 , 26) ); -} - - -/* we packed 256 7-bit values, touching 7 256-bit words, using 112 bytes */ -static void avxunpackblock7(const __m256i * compressed, uint32_t * pout) { - /* we are going to access 7 256-bit words */ - __m256i w0, w1; - __m256i * out = (__m256i *) pout; - const __m256i mask = _mm256_set1_epi32(127); - w0 = _mm256_lddqu_si256 (compressed); - _mm256_storeu_si256(out + 0, _mm256_and_si256 ( mask, w0 ) ); - _mm256_storeu_si256(out + 1, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 7) ) ); - _mm256_storeu_si256(out + 2, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 14) ) ); - _mm256_storeu_si256(out + 3, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 21) ) ); - w1 = _mm256_lddqu_si256 (compressed + 1); - _mm256_storeu_si256(out + 4, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 28) ,_mm256_slli_epi32( w1 , 4 ) ) ) ); - _mm256_storeu_si256(out + 5, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 3) ) ); - _mm256_storeu_si256(out + 6, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 10) ) ); - _mm256_storeu_si256(out + 7, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 17) ) ); - _mm256_storeu_si256(out + 8, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 24) ) ); - w0 = _mm256_lddqu_si256 (compressed + 2); - _mm256_storeu_si256(out + 9, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 31) ,_mm256_slli_epi32( w0 , 1 ) ) ) ); - _mm256_storeu_si256(out + 10, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 6) ) ); - _mm256_storeu_si256(out + 11, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 13) ) ); - _mm256_storeu_si256(out + 12, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 20) ) ); - w1 = _mm256_lddqu_si256 (compressed + 3); - _mm256_storeu_si256(out + 13, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 27) ,_mm256_slli_epi32( w1 , 5 ) ) ) ); - _mm256_storeu_si256(out + 14, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 2) ) ); - _mm256_storeu_si256(out + 15, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 9) ) ); - _mm256_storeu_si256(out + 16, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 16) ) ); - _mm256_storeu_si256(out + 17, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 23) ) ); - w0 = _mm256_lddqu_si256 (compressed + 4); - _mm256_storeu_si256(out + 18, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 30) ,_mm256_slli_epi32( w0 , 2 ) ) ) ); - _mm256_storeu_si256(out + 19, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 5) ) ); - _mm256_storeu_si256(out + 20, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 12) ) ); - _mm256_storeu_si256(out + 21, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 19) ) ); - w1 = _mm256_lddqu_si256 (compressed + 5); - _mm256_storeu_si256(out + 22, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 26) ,_mm256_slli_epi32( w1 , 6 ) ) ) ); - _mm256_storeu_si256(out + 23, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 1) ) ); - _mm256_storeu_si256(out + 24, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 8) ) ); - _mm256_storeu_si256(out + 25, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 15) ) ); - _mm256_storeu_si256(out + 26, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 22) ) ); - w0 = _mm256_lddqu_si256 (compressed + 6); - _mm256_storeu_si256(out + 27, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 29) ,_mm256_slli_epi32( w0 , 3 ) ) ) ); - _mm256_storeu_si256(out + 28, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 4) ) ); - _mm256_storeu_si256(out + 29, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 11) ) ); - _mm256_storeu_si256(out + 30, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 18) ) ); - _mm256_storeu_si256(out + 31, _mm256_srli_epi32( w0 , 25) ); -} - - -/* we packed 256 8-bit values, touching 8 256-bit words, using 128 bytes */ -static void avxunpackblock8(const __m256i * compressed, uint32_t * pout) { - /* we are going to access 8 256-bit words */ - __m256i w0, w1; - __m256i * out = (__m256i *) pout; - const __m256i mask = _mm256_set1_epi32(255); - w0 = _mm256_lddqu_si256 (compressed); - _mm256_storeu_si256(out + 0, _mm256_and_si256 ( mask, w0 ) ); - _mm256_storeu_si256(out + 1, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 8) ) ); - _mm256_storeu_si256(out + 2, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 16) ) ); - _mm256_storeu_si256(out + 3, _mm256_srli_epi32( w0 , 24) ); - w1 = _mm256_lddqu_si256 (compressed + 1); - _mm256_storeu_si256(out + 4, _mm256_and_si256 ( mask, w1 ) ); - _mm256_storeu_si256(out + 5, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 8) ) ); - _mm256_storeu_si256(out + 6, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 16) ) ); - _mm256_storeu_si256(out + 7, _mm256_srli_epi32( w1 , 24) ); - w0 = _mm256_lddqu_si256 (compressed + 2); - _mm256_storeu_si256(out + 8, _mm256_and_si256 ( mask, w0 ) ); - _mm256_storeu_si256(out + 9, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 8) ) ); - _mm256_storeu_si256(out + 10, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 16) ) ); - _mm256_storeu_si256(out + 11, _mm256_srli_epi32( w0 , 24) ); - w1 = _mm256_lddqu_si256 (compressed + 3); - _mm256_storeu_si256(out + 12, _mm256_and_si256 ( mask, w1 ) ); - _mm256_storeu_si256(out + 13, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 8) ) ); - _mm256_storeu_si256(out + 14, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 16) ) ); - _mm256_storeu_si256(out + 15, _mm256_srli_epi32( w1 , 24) ); - w0 = _mm256_lddqu_si256 (compressed + 4); - _mm256_storeu_si256(out + 16, _mm256_and_si256 ( mask, w0 ) ); - _mm256_storeu_si256(out + 17, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 8) ) ); - _mm256_storeu_si256(out + 18, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 16) ) ); - _mm256_storeu_si256(out + 19, _mm256_srli_epi32( w0 , 24) ); - w1 = _mm256_lddqu_si256 (compressed + 5); - _mm256_storeu_si256(out + 20, _mm256_and_si256 ( mask, w1 ) ); - _mm256_storeu_si256(out + 21, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 8) ) ); - _mm256_storeu_si256(out + 22, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 16) ) ); - _mm256_storeu_si256(out + 23, _mm256_srli_epi32( w1 , 24) ); - w0 = _mm256_lddqu_si256 (compressed + 6); - _mm256_storeu_si256(out + 24, _mm256_and_si256 ( mask, w0 ) ); - _mm256_storeu_si256(out + 25, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 8) ) ); - _mm256_storeu_si256(out + 26, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 16) ) ); - _mm256_storeu_si256(out + 27, _mm256_srli_epi32( w0 , 24) ); - w1 = _mm256_lddqu_si256 (compressed + 7); - _mm256_storeu_si256(out + 28, _mm256_and_si256 ( mask, w1 ) ); - _mm256_storeu_si256(out + 29, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 8) ) ); - _mm256_storeu_si256(out + 30, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 16) ) ); - _mm256_storeu_si256(out + 31, _mm256_srli_epi32( w1 , 24) ); -} - - -/* we packed 256 9-bit values, touching 9 256-bit words, using 144 bytes */ -static void avxunpackblock9(const __m256i * compressed, uint32_t * pout) { - /* we are going to access 9 256-bit words */ - __m256i w0, w1; - __m256i * out = (__m256i *) pout; - const __m256i mask = _mm256_set1_epi32(511); - w0 = _mm256_lddqu_si256 (compressed); - _mm256_storeu_si256(out + 0, _mm256_and_si256 ( mask, w0 ) ); - _mm256_storeu_si256(out + 1, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 9) ) ); - _mm256_storeu_si256(out + 2, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 18) ) ); - w1 = _mm256_lddqu_si256 (compressed + 1); - _mm256_storeu_si256(out + 3, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 27) ,_mm256_slli_epi32( w1 , 5 ) ) ) ); - _mm256_storeu_si256(out + 4, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 4) ) ); - _mm256_storeu_si256(out + 5, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 13) ) ); - _mm256_storeu_si256(out + 6, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 22) ) ); - w0 = _mm256_lddqu_si256 (compressed + 2); - _mm256_storeu_si256(out + 7, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 31) ,_mm256_slli_epi32( w0 , 1 ) ) ) ); - _mm256_storeu_si256(out + 8, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 8) ) ); - _mm256_storeu_si256(out + 9, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 17) ) ); - w1 = _mm256_lddqu_si256 (compressed + 3); - _mm256_storeu_si256(out + 10, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 26) ,_mm256_slli_epi32( w1 , 6 ) ) ) ); - _mm256_storeu_si256(out + 11, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 3) ) ); - _mm256_storeu_si256(out + 12, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 12) ) ); - _mm256_storeu_si256(out + 13, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 21) ) ); - w0 = _mm256_lddqu_si256 (compressed + 4); - _mm256_storeu_si256(out + 14, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 30) ,_mm256_slli_epi32( w0 , 2 ) ) ) ); - _mm256_storeu_si256(out + 15, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 7) ) ); - _mm256_storeu_si256(out + 16, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 16) ) ); - w1 = _mm256_lddqu_si256 (compressed + 5); - _mm256_storeu_si256(out + 17, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 25) ,_mm256_slli_epi32( w1 , 7 ) ) ) ); - _mm256_storeu_si256(out + 18, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 2) ) ); - _mm256_storeu_si256(out + 19, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 11) ) ); - _mm256_storeu_si256(out + 20, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 20) ) ); - w0 = _mm256_lddqu_si256 (compressed + 6); - _mm256_storeu_si256(out + 21, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 29) ,_mm256_slli_epi32( w0 , 3 ) ) ) ); - _mm256_storeu_si256(out + 22, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 6) ) ); - _mm256_storeu_si256(out + 23, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 15) ) ); - w1 = _mm256_lddqu_si256 (compressed + 7); - _mm256_storeu_si256(out + 24, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 24) ,_mm256_slli_epi32( w1 , 8 ) ) ) ); - _mm256_storeu_si256(out + 25, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 1) ) ); - _mm256_storeu_si256(out + 26, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 10) ) ); - _mm256_storeu_si256(out + 27, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 19) ) ); - w0 = _mm256_lddqu_si256 (compressed + 8); - _mm256_storeu_si256(out + 28, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 28) ,_mm256_slli_epi32( w0 , 4 ) ) ) ); - _mm256_storeu_si256(out + 29, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 5) ) ); - _mm256_storeu_si256(out + 30, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 14) ) ); - _mm256_storeu_si256(out + 31, _mm256_srli_epi32( w0 , 23) ); -} - - -/* we packed 256 10-bit values, touching 10 256-bit words, using 160 bytes */ -static void avxunpackblock10(const __m256i * compressed, uint32_t * pout) { - /* we are going to access 10 256-bit words */ - __m256i w0, w1; - __m256i * out = (__m256i *) pout; - const __m256i mask = _mm256_set1_epi32(1023); - w0 = _mm256_lddqu_si256 (compressed); - _mm256_storeu_si256(out + 0, _mm256_and_si256 ( mask, w0 ) ); - _mm256_storeu_si256(out + 1, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 10) ) ); - _mm256_storeu_si256(out + 2, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 20) ) ); - w1 = _mm256_lddqu_si256 (compressed + 1); - _mm256_storeu_si256(out + 3, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 30) ,_mm256_slli_epi32( w1 , 2 ) ) ) ); - _mm256_storeu_si256(out + 4, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 8) ) ); - _mm256_storeu_si256(out + 5, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 18) ) ); - w0 = _mm256_lddqu_si256 (compressed + 2); - _mm256_storeu_si256(out + 6, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 28) ,_mm256_slli_epi32( w0 , 4 ) ) ) ); - _mm256_storeu_si256(out + 7, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 6) ) ); - _mm256_storeu_si256(out + 8, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 16) ) ); - w1 = _mm256_lddqu_si256 (compressed + 3); - _mm256_storeu_si256(out + 9, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 26) ,_mm256_slli_epi32( w1 , 6 ) ) ) ); - _mm256_storeu_si256(out + 10, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 4) ) ); - _mm256_storeu_si256(out + 11, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 14) ) ); - w0 = _mm256_lddqu_si256 (compressed + 4); - _mm256_storeu_si256(out + 12, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 24) ,_mm256_slli_epi32( w0 , 8 ) ) ) ); - _mm256_storeu_si256(out + 13, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 2) ) ); - _mm256_storeu_si256(out + 14, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 12) ) ); - _mm256_storeu_si256(out + 15, _mm256_srli_epi32( w0 , 22) ); - w1 = _mm256_lddqu_si256 (compressed + 5); - _mm256_storeu_si256(out + 16, _mm256_and_si256 ( mask, w1 ) ); - _mm256_storeu_si256(out + 17, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 10) ) ); - _mm256_storeu_si256(out + 18, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 20) ) ); - w0 = _mm256_lddqu_si256 (compressed + 6); - _mm256_storeu_si256(out + 19, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 30) ,_mm256_slli_epi32( w0 , 2 ) ) ) ); - _mm256_storeu_si256(out + 20, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 8) ) ); - _mm256_storeu_si256(out + 21, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 18) ) ); - w1 = _mm256_lddqu_si256 (compressed + 7); - _mm256_storeu_si256(out + 22, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 28) ,_mm256_slli_epi32( w1 , 4 ) ) ) ); - _mm256_storeu_si256(out + 23, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 6) ) ); - _mm256_storeu_si256(out + 24, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 16) ) ); - w0 = _mm256_lddqu_si256 (compressed + 8); - _mm256_storeu_si256(out + 25, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 26) ,_mm256_slli_epi32( w0 , 6 ) ) ) ); - _mm256_storeu_si256(out + 26, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 4) ) ); - _mm256_storeu_si256(out + 27, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 14) ) ); - w1 = _mm256_lddqu_si256 (compressed + 9); - _mm256_storeu_si256(out + 28, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 24) ,_mm256_slli_epi32( w1 , 8 ) ) ) ); - _mm256_storeu_si256(out + 29, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 2) ) ); - _mm256_storeu_si256(out + 30, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 12) ) ); - _mm256_storeu_si256(out + 31, _mm256_srli_epi32( w1 , 22) ); -} - - -/* we packed 256 11-bit values, touching 11 256-bit words, using 176 bytes */ -static void avxunpackblock11(const __m256i * compressed, uint32_t * pout) { - /* we are going to access 11 256-bit words */ - __m256i w0, w1; - __m256i * out = (__m256i *) pout; - const __m256i mask = _mm256_set1_epi32(2047); - w0 = _mm256_lddqu_si256 (compressed); - _mm256_storeu_si256(out + 0, _mm256_and_si256 ( mask, w0 ) ); - _mm256_storeu_si256(out + 1, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 11) ) ); - w1 = _mm256_lddqu_si256 (compressed + 1); - _mm256_storeu_si256(out + 2, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 22) ,_mm256_slli_epi32( w1 , 10 ) ) ) ); - _mm256_storeu_si256(out + 3, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 1) ) ); - _mm256_storeu_si256(out + 4, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 12) ) ); - w0 = _mm256_lddqu_si256 (compressed + 2); - _mm256_storeu_si256(out + 5, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 23) ,_mm256_slli_epi32( w0 , 9 ) ) ) ); - _mm256_storeu_si256(out + 6, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 2) ) ); - _mm256_storeu_si256(out + 7, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 13) ) ); - w1 = _mm256_lddqu_si256 (compressed + 3); - _mm256_storeu_si256(out + 8, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 24) ,_mm256_slli_epi32( w1 , 8 ) ) ) ); - _mm256_storeu_si256(out + 9, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 3) ) ); - _mm256_storeu_si256(out + 10, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 14) ) ); - w0 = _mm256_lddqu_si256 (compressed + 4); - _mm256_storeu_si256(out + 11, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 25) ,_mm256_slli_epi32( w0 , 7 ) ) ) ); - _mm256_storeu_si256(out + 12, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 4) ) ); - _mm256_storeu_si256(out + 13, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 15) ) ); - w1 = _mm256_lddqu_si256 (compressed + 5); - _mm256_storeu_si256(out + 14, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 26) ,_mm256_slli_epi32( w1 , 6 ) ) ) ); - _mm256_storeu_si256(out + 15, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 5) ) ); - _mm256_storeu_si256(out + 16, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 16) ) ); - w0 = _mm256_lddqu_si256 (compressed + 6); - _mm256_storeu_si256(out + 17, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 27) ,_mm256_slli_epi32( w0 , 5 ) ) ) ); - _mm256_storeu_si256(out + 18, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 6) ) ); - _mm256_storeu_si256(out + 19, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 17) ) ); - w1 = _mm256_lddqu_si256 (compressed + 7); - _mm256_storeu_si256(out + 20, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 28) ,_mm256_slli_epi32( w1 , 4 ) ) ) ); - _mm256_storeu_si256(out + 21, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 7) ) ); - _mm256_storeu_si256(out + 22, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 18) ) ); - w0 = _mm256_lddqu_si256 (compressed + 8); - _mm256_storeu_si256(out + 23, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 29) ,_mm256_slli_epi32( w0 , 3 ) ) ) ); - _mm256_storeu_si256(out + 24, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 8) ) ); - _mm256_storeu_si256(out + 25, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 19) ) ); - w1 = _mm256_lddqu_si256 (compressed + 9); - _mm256_storeu_si256(out + 26, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 30) ,_mm256_slli_epi32( w1 , 2 ) ) ) ); - _mm256_storeu_si256(out + 27, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 9) ) ); - _mm256_storeu_si256(out + 28, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 20) ) ); - w0 = _mm256_lddqu_si256 (compressed + 10); - _mm256_storeu_si256(out + 29, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 31) ,_mm256_slli_epi32( w0 , 1 ) ) ) ); - _mm256_storeu_si256(out + 30, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 10) ) ); - _mm256_storeu_si256(out + 31, _mm256_srli_epi32( w0 , 21) ); -} - - -/* we packed 256 12-bit values, touching 12 256-bit words, using 192 bytes */ -static void avxunpackblock12(const __m256i * compressed, uint32_t * pout) { - /* we are going to access 12 256-bit words */ - __m256i w0, w1; - __m256i * out = (__m256i *) pout; - const __m256i mask = _mm256_set1_epi32(4095); - w0 = _mm256_lddqu_si256 (compressed); - _mm256_storeu_si256(out + 0, _mm256_and_si256 ( mask, w0 ) ); - _mm256_storeu_si256(out + 1, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 12) ) ); - w1 = _mm256_lddqu_si256 (compressed + 1); - _mm256_storeu_si256(out + 2, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 24) ,_mm256_slli_epi32( w1 , 8 ) ) ) ); - _mm256_storeu_si256(out + 3, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 4) ) ); - _mm256_storeu_si256(out + 4, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 16) ) ); - w0 = _mm256_lddqu_si256 (compressed + 2); - _mm256_storeu_si256(out + 5, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 28) ,_mm256_slli_epi32( w0 , 4 ) ) ) ); - _mm256_storeu_si256(out + 6, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 8) ) ); - _mm256_storeu_si256(out + 7, _mm256_srli_epi32( w0 , 20) ); - w1 = _mm256_lddqu_si256 (compressed + 3); - _mm256_storeu_si256(out + 8, _mm256_and_si256 ( mask, w1 ) ); - _mm256_storeu_si256(out + 9, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 12) ) ); - w0 = _mm256_lddqu_si256 (compressed + 4); - _mm256_storeu_si256(out + 10, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 24) ,_mm256_slli_epi32( w0 , 8 ) ) ) ); - _mm256_storeu_si256(out + 11, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 4) ) ); - _mm256_storeu_si256(out + 12, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 16) ) ); - w1 = _mm256_lddqu_si256 (compressed + 5); - _mm256_storeu_si256(out + 13, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 28) ,_mm256_slli_epi32( w1 , 4 ) ) ) ); - _mm256_storeu_si256(out + 14, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 8) ) ); - _mm256_storeu_si256(out + 15, _mm256_srli_epi32( w1 , 20) ); - w0 = _mm256_lddqu_si256 (compressed + 6); - _mm256_storeu_si256(out + 16, _mm256_and_si256 ( mask, w0 ) ); - _mm256_storeu_si256(out + 17, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 12) ) ); - w1 = _mm256_lddqu_si256 (compressed + 7); - _mm256_storeu_si256(out + 18, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 24) ,_mm256_slli_epi32( w1 , 8 ) ) ) ); - _mm256_storeu_si256(out + 19, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 4) ) ); - _mm256_storeu_si256(out + 20, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 16) ) ); - w0 = _mm256_lddqu_si256 (compressed + 8); - _mm256_storeu_si256(out + 21, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 28) ,_mm256_slli_epi32( w0 , 4 ) ) ) ); - _mm256_storeu_si256(out + 22, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 8) ) ); - _mm256_storeu_si256(out + 23, _mm256_srli_epi32( w0 , 20) ); - w1 = _mm256_lddqu_si256 (compressed + 9); - _mm256_storeu_si256(out + 24, _mm256_and_si256 ( mask, w1 ) ); - _mm256_storeu_si256(out + 25, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 12) ) ); - w0 = _mm256_lddqu_si256 (compressed + 10); - _mm256_storeu_si256(out + 26, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 24) ,_mm256_slli_epi32( w0 , 8 ) ) ) ); - _mm256_storeu_si256(out + 27, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 4) ) ); - _mm256_storeu_si256(out + 28, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 16) ) ); - w1 = _mm256_lddqu_si256 (compressed + 11); - _mm256_storeu_si256(out + 29, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 28) ,_mm256_slli_epi32( w1 , 4 ) ) ) ); - _mm256_storeu_si256(out + 30, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 8) ) ); - _mm256_storeu_si256(out + 31, _mm256_srli_epi32( w1 , 20) ); -} - - -/* we packed 256 13-bit values, touching 13 256-bit words, using 208 bytes */ -static void avxunpackblock13(const __m256i * compressed, uint32_t * pout) { - /* we are going to access 13 256-bit words */ - __m256i w0, w1; - __m256i * out = (__m256i *) pout; - const __m256i mask = _mm256_set1_epi32(8191); - w0 = _mm256_lddqu_si256 (compressed); - _mm256_storeu_si256(out + 0, _mm256_and_si256 ( mask, w0 ) ); - _mm256_storeu_si256(out + 1, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 13) ) ); - w1 = _mm256_lddqu_si256 (compressed + 1); - _mm256_storeu_si256(out + 2, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 26) ,_mm256_slli_epi32( w1 , 6 ) ) ) ); - _mm256_storeu_si256(out + 3, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 7) ) ); - w0 = _mm256_lddqu_si256 (compressed + 2); - _mm256_storeu_si256(out + 4, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 20) ,_mm256_slli_epi32( w0 , 12 ) ) ) ); - _mm256_storeu_si256(out + 5, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 1) ) ); - _mm256_storeu_si256(out + 6, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 14) ) ); - w1 = _mm256_lddqu_si256 (compressed + 3); - _mm256_storeu_si256(out + 7, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 27) ,_mm256_slli_epi32( w1 , 5 ) ) ) ); - _mm256_storeu_si256(out + 8, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 8) ) ); - w0 = _mm256_lddqu_si256 (compressed + 4); - _mm256_storeu_si256(out + 9, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 21) ,_mm256_slli_epi32( w0 , 11 ) ) ) ); - _mm256_storeu_si256(out + 10, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 2) ) ); - _mm256_storeu_si256(out + 11, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 15) ) ); - w1 = _mm256_lddqu_si256 (compressed + 5); - _mm256_storeu_si256(out + 12, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 28) ,_mm256_slli_epi32( w1 , 4 ) ) ) ); - _mm256_storeu_si256(out + 13, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 9) ) ); - w0 = _mm256_lddqu_si256 (compressed + 6); - _mm256_storeu_si256(out + 14, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 22) ,_mm256_slli_epi32( w0 , 10 ) ) ) ); - _mm256_storeu_si256(out + 15, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 3) ) ); - _mm256_storeu_si256(out + 16, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 16) ) ); - w1 = _mm256_lddqu_si256 (compressed + 7); - _mm256_storeu_si256(out + 17, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 29) ,_mm256_slli_epi32( w1 , 3 ) ) ) ); - _mm256_storeu_si256(out + 18, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 10) ) ); - w0 = _mm256_lddqu_si256 (compressed + 8); - _mm256_storeu_si256(out + 19, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 23) ,_mm256_slli_epi32( w0 , 9 ) ) ) ); - _mm256_storeu_si256(out + 20, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 4) ) ); - _mm256_storeu_si256(out + 21, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 17) ) ); - w1 = _mm256_lddqu_si256 (compressed + 9); - _mm256_storeu_si256(out + 22, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 30) ,_mm256_slli_epi32( w1 , 2 ) ) ) ); - _mm256_storeu_si256(out + 23, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 11) ) ); - w0 = _mm256_lddqu_si256 (compressed + 10); - _mm256_storeu_si256(out + 24, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 24) ,_mm256_slli_epi32( w0 , 8 ) ) ) ); - _mm256_storeu_si256(out + 25, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 5) ) ); - _mm256_storeu_si256(out + 26, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 18) ) ); - w1 = _mm256_lddqu_si256 (compressed + 11); - _mm256_storeu_si256(out + 27, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 31) ,_mm256_slli_epi32( w1 , 1 ) ) ) ); - _mm256_storeu_si256(out + 28, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 12) ) ); - w0 = _mm256_lddqu_si256 (compressed + 12); - _mm256_storeu_si256(out + 29, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 25) ,_mm256_slli_epi32( w0 , 7 ) ) ) ); - _mm256_storeu_si256(out + 30, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 6) ) ); - _mm256_storeu_si256(out + 31, _mm256_srli_epi32( w0 , 19) ); -} - - -/* we packed 256 14-bit values, touching 14 256-bit words, using 224 bytes */ -static void avxunpackblock14(const __m256i * compressed, uint32_t * pout) { - /* we are going to access 14 256-bit words */ - __m256i w0, w1; - __m256i * out = (__m256i *) pout; - const __m256i mask = _mm256_set1_epi32(16383); - w0 = _mm256_lddqu_si256 (compressed); - _mm256_storeu_si256(out + 0, _mm256_and_si256 ( mask, w0 ) ); - _mm256_storeu_si256(out + 1, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 14) ) ); - w1 = _mm256_lddqu_si256 (compressed + 1); - _mm256_storeu_si256(out + 2, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 28) ,_mm256_slli_epi32( w1 , 4 ) ) ) ); - _mm256_storeu_si256(out + 3, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 10) ) ); - w0 = _mm256_lddqu_si256 (compressed + 2); - _mm256_storeu_si256(out + 4, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 24) ,_mm256_slli_epi32( w0 , 8 ) ) ) ); - _mm256_storeu_si256(out + 5, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 6) ) ); - w1 = _mm256_lddqu_si256 (compressed + 3); - _mm256_storeu_si256(out + 6, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 20) ,_mm256_slli_epi32( w1 , 12 ) ) ) ); - _mm256_storeu_si256(out + 7, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 2) ) ); - _mm256_storeu_si256(out + 8, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 16) ) ); - w0 = _mm256_lddqu_si256 (compressed + 4); - _mm256_storeu_si256(out + 9, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 30) ,_mm256_slli_epi32( w0 , 2 ) ) ) ); - _mm256_storeu_si256(out + 10, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 12) ) ); - w1 = _mm256_lddqu_si256 (compressed + 5); - _mm256_storeu_si256(out + 11, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 26) ,_mm256_slli_epi32( w1 , 6 ) ) ) ); - _mm256_storeu_si256(out + 12, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 8) ) ); - w0 = _mm256_lddqu_si256 (compressed + 6); - _mm256_storeu_si256(out + 13, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 22) ,_mm256_slli_epi32( w0 , 10 ) ) ) ); - _mm256_storeu_si256(out + 14, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 4) ) ); - _mm256_storeu_si256(out + 15, _mm256_srli_epi32( w0 , 18) ); - w1 = _mm256_lddqu_si256 (compressed + 7); - _mm256_storeu_si256(out + 16, _mm256_and_si256 ( mask, w1 ) ); - _mm256_storeu_si256(out + 17, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 14) ) ); - w0 = _mm256_lddqu_si256 (compressed + 8); - _mm256_storeu_si256(out + 18, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 28) ,_mm256_slli_epi32( w0 , 4 ) ) ) ); - _mm256_storeu_si256(out + 19, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 10) ) ); - w1 = _mm256_lddqu_si256 (compressed + 9); - _mm256_storeu_si256(out + 20, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 24) ,_mm256_slli_epi32( w1 , 8 ) ) ) ); - _mm256_storeu_si256(out + 21, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 6) ) ); - w0 = _mm256_lddqu_si256 (compressed + 10); - _mm256_storeu_si256(out + 22, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 20) ,_mm256_slli_epi32( w0 , 12 ) ) ) ); - _mm256_storeu_si256(out + 23, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 2) ) ); - _mm256_storeu_si256(out + 24, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 16) ) ); - w1 = _mm256_lddqu_si256 (compressed + 11); - _mm256_storeu_si256(out + 25, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 30) ,_mm256_slli_epi32( w1 , 2 ) ) ) ); - _mm256_storeu_si256(out + 26, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 12) ) ); - w0 = _mm256_lddqu_si256 (compressed + 12); - _mm256_storeu_si256(out + 27, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 26) ,_mm256_slli_epi32( w0 , 6 ) ) ) ); - _mm256_storeu_si256(out + 28, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 8) ) ); - w1 = _mm256_lddqu_si256 (compressed + 13); - _mm256_storeu_si256(out + 29, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 22) ,_mm256_slli_epi32( w1 , 10 ) ) ) ); - _mm256_storeu_si256(out + 30, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 4) ) ); - _mm256_storeu_si256(out + 31, _mm256_srli_epi32( w1 , 18) ); -} - - -/* we packed 256 15-bit values, touching 15 256-bit words, using 240 bytes */ -static void avxunpackblock15(const __m256i * compressed, uint32_t * pout) { - /* we are going to access 15 256-bit words */ - __m256i w0, w1; - __m256i * out = (__m256i *) pout; - const __m256i mask = _mm256_set1_epi32(32767); - w0 = _mm256_lddqu_si256 (compressed); - _mm256_storeu_si256(out + 0, _mm256_and_si256 ( mask, w0 ) ); - _mm256_storeu_si256(out + 1, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 15) ) ); - w1 = _mm256_lddqu_si256 (compressed + 1); - _mm256_storeu_si256(out + 2, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 30) ,_mm256_slli_epi32( w1 , 2 ) ) ) ); - _mm256_storeu_si256(out + 3, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 13) ) ); - w0 = _mm256_lddqu_si256 (compressed + 2); - _mm256_storeu_si256(out + 4, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 28) ,_mm256_slli_epi32( w0 , 4 ) ) ) ); - _mm256_storeu_si256(out + 5, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 11) ) ); - w1 = _mm256_lddqu_si256 (compressed + 3); - _mm256_storeu_si256(out + 6, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 26) ,_mm256_slli_epi32( w1 , 6 ) ) ) ); - _mm256_storeu_si256(out + 7, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 9) ) ); - w0 = _mm256_lddqu_si256 (compressed + 4); - _mm256_storeu_si256(out + 8, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 24) ,_mm256_slli_epi32( w0 , 8 ) ) ) ); - _mm256_storeu_si256(out + 9, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 7) ) ); - w1 = _mm256_lddqu_si256 (compressed + 5); - _mm256_storeu_si256(out + 10, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 22) ,_mm256_slli_epi32( w1 , 10 ) ) ) ); - _mm256_storeu_si256(out + 11, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 5) ) ); - w0 = _mm256_lddqu_si256 (compressed + 6); - _mm256_storeu_si256(out + 12, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 20) ,_mm256_slli_epi32( w0 , 12 ) ) ) ); - _mm256_storeu_si256(out + 13, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 3) ) ); - w1 = _mm256_lddqu_si256 (compressed + 7); - _mm256_storeu_si256(out + 14, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 18) ,_mm256_slli_epi32( w1 , 14 ) ) ) ); - _mm256_storeu_si256(out + 15, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 1) ) ); - _mm256_storeu_si256(out + 16, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 16) ) ); - w0 = _mm256_lddqu_si256 (compressed + 8); - _mm256_storeu_si256(out + 17, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 31) ,_mm256_slli_epi32( w0 , 1 ) ) ) ); - _mm256_storeu_si256(out + 18, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 14) ) ); - w1 = _mm256_lddqu_si256 (compressed + 9); - _mm256_storeu_si256(out + 19, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 29) ,_mm256_slli_epi32( w1 , 3 ) ) ) ); - _mm256_storeu_si256(out + 20, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 12) ) ); - w0 = _mm256_lddqu_si256 (compressed + 10); - _mm256_storeu_si256(out + 21, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 27) ,_mm256_slli_epi32( w0 , 5 ) ) ) ); - _mm256_storeu_si256(out + 22, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 10) ) ); - w1 = _mm256_lddqu_si256 (compressed + 11); - _mm256_storeu_si256(out + 23, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 25) ,_mm256_slli_epi32( w1 , 7 ) ) ) ); - _mm256_storeu_si256(out + 24, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 8) ) ); - w0 = _mm256_lddqu_si256 (compressed + 12); - _mm256_storeu_si256(out + 25, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 23) ,_mm256_slli_epi32( w0 , 9 ) ) ) ); - _mm256_storeu_si256(out + 26, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 6) ) ); - w1 = _mm256_lddqu_si256 (compressed + 13); - _mm256_storeu_si256(out + 27, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 21) ,_mm256_slli_epi32( w1 , 11 ) ) ) ); - _mm256_storeu_si256(out + 28, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 4) ) ); - w0 = _mm256_lddqu_si256 (compressed + 14); - _mm256_storeu_si256(out + 29, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 19) ,_mm256_slli_epi32( w0 , 13 ) ) ) ); - _mm256_storeu_si256(out + 30, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 2) ) ); - _mm256_storeu_si256(out + 31, _mm256_srli_epi32( w0 , 17) ); -} - - -/* we packed 256 16-bit values, touching 16 256-bit words, using 256 bytes */ -static void avxunpackblock16(const __m256i * compressed, uint32_t * pout) { - /* we are going to access 16 256-bit words */ - __m256i w0, w1; - __m256i * out = (__m256i *) pout; - const __m256i mask = _mm256_set1_epi32(65535); - w0 = _mm256_lddqu_si256 (compressed); - _mm256_storeu_si256(out + 0, _mm256_and_si256 ( mask, w0 ) ); - _mm256_storeu_si256(out + 1, _mm256_srli_epi32( w0 , 16) ); - w1 = _mm256_lddqu_si256 (compressed + 1); - _mm256_storeu_si256(out + 2, _mm256_and_si256 ( mask, w1 ) ); - _mm256_storeu_si256(out + 3, _mm256_srli_epi32( w1 , 16) ); - w0 = _mm256_lddqu_si256 (compressed + 2); - _mm256_storeu_si256(out + 4, _mm256_and_si256 ( mask, w0 ) ); - _mm256_storeu_si256(out + 5, _mm256_srli_epi32( w0 , 16) ); - w1 = _mm256_lddqu_si256 (compressed + 3); - _mm256_storeu_si256(out + 6, _mm256_and_si256 ( mask, w1 ) ); - _mm256_storeu_si256(out + 7, _mm256_srli_epi32( w1 , 16) ); - w0 = _mm256_lddqu_si256 (compressed + 4); - _mm256_storeu_si256(out + 8, _mm256_and_si256 ( mask, w0 ) ); - _mm256_storeu_si256(out + 9, _mm256_srli_epi32( w0 , 16) ); - w1 = _mm256_lddqu_si256 (compressed + 5); - _mm256_storeu_si256(out + 10, _mm256_and_si256 ( mask, w1 ) ); - _mm256_storeu_si256(out + 11, _mm256_srli_epi32( w1 , 16) ); - w0 = _mm256_lddqu_si256 (compressed + 6); - _mm256_storeu_si256(out + 12, _mm256_and_si256 ( mask, w0 ) ); - _mm256_storeu_si256(out + 13, _mm256_srli_epi32( w0 , 16) ); - w1 = _mm256_lddqu_si256 (compressed + 7); - _mm256_storeu_si256(out + 14, _mm256_and_si256 ( mask, w1 ) ); - _mm256_storeu_si256(out + 15, _mm256_srli_epi32( w1 , 16) ); - w0 = _mm256_lddqu_si256 (compressed + 8); - _mm256_storeu_si256(out + 16, _mm256_and_si256 ( mask, w0 ) ); - _mm256_storeu_si256(out + 17, _mm256_srli_epi32( w0 , 16) ); - w1 = _mm256_lddqu_si256 (compressed + 9); - _mm256_storeu_si256(out + 18, _mm256_and_si256 ( mask, w1 ) ); - _mm256_storeu_si256(out + 19, _mm256_srli_epi32( w1 , 16) ); - w0 = _mm256_lddqu_si256 (compressed + 10); - _mm256_storeu_si256(out + 20, _mm256_and_si256 ( mask, w0 ) ); - _mm256_storeu_si256(out + 21, _mm256_srli_epi32( w0 , 16) ); - w1 = _mm256_lddqu_si256 (compressed + 11); - _mm256_storeu_si256(out + 22, _mm256_and_si256 ( mask, w1 ) ); - _mm256_storeu_si256(out + 23, _mm256_srli_epi32( w1 , 16) ); - w0 = _mm256_lddqu_si256 (compressed + 12); - _mm256_storeu_si256(out + 24, _mm256_and_si256 ( mask, w0 ) ); - _mm256_storeu_si256(out + 25, _mm256_srli_epi32( w0 , 16) ); - w1 = _mm256_lddqu_si256 (compressed + 13); - _mm256_storeu_si256(out + 26, _mm256_and_si256 ( mask, w1 ) ); - _mm256_storeu_si256(out + 27, _mm256_srli_epi32( w1 , 16) ); - w0 = _mm256_lddqu_si256 (compressed + 14); - _mm256_storeu_si256(out + 28, _mm256_and_si256 ( mask, w0 ) ); - _mm256_storeu_si256(out + 29, _mm256_srli_epi32( w0 , 16) ); - w1 = _mm256_lddqu_si256 (compressed + 15); - _mm256_storeu_si256(out + 30, _mm256_and_si256 ( mask, w1 ) ); - _mm256_storeu_si256(out + 31, _mm256_srli_epi32( w1 , 16) ); -} - - -/* we packed 256 17-bit values, touching 17 256-bit words, using 272 bytes */ -static void avxunpackblock17(const __m256i * compressed, uint32_t * pout) { - /* we are going to access 17 256-bit words */ - __m256i w0, w1; - __m256i * out = (__m256i *) pout; - const __m256i mask = _mm256_set1_epi32(131071); - w0 = _mm256_lddqu_si256 (compressed); - _mm256_storeu_si256(out + 0, _mm256_and_si256 ( mask, w0 ) ); - w1 = _mm256_lddqu_si256 (compressed + 1); - _mm256_storeu_si256(out + 1, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 17) ,_mm256_slli_epi32( w1 , 15 ) ) ) ); - _mm256_storeu_si256(out + 2, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 2) ) ); - w0 = _mm256_lddqu_si256 (compressed + 2); - _mm256_storeu_si256(out + 3, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 19) ,_mm256_slli_epi32( w0 , 13 ) ) ) ); - _mm256_storeu_si256(out + 4, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 4) ) ); - w1 = _mm256_lddqu_si256 (compressed + 3); - _mm256_storeu_si256(out + 5, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 21) ,_mm256_slli_epi32( w1 , 11 ) ) ) ); - _mm256_storeu_si256(out + 6, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 6) ) ); - w0 = _mm256_lddqu_si256 (compressed + 4); - _mm256_storeu_si256(out + 7, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 23) ,_mm256_slli_epi32( w0 , 9 ) ) ) ); - _mm256_storeu_si256(out + 8, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 8) ) ); - w1 = _mm256_lddqu_si256 (compressed + 5); - _mm256_storeu_si256(out + 9, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 25) ,_mm256_slli_epi32( w1 , 7 ) ) ) ); - _mm256_storeu_si256(out + 10, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 10) ) ); - w0 = _mm256_lddqu_si256 (compressed + 6); - _mm256_storeu_si256(out + 11, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 27) ,_mm256_slli_epi32( w0 , 5 ) ) ) ); - _mm256_storeu_si256(out + 12, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 12) ) ); - w1 = _mm256_lddqu_si256 (compressed + 7); - _mm256_storeu_si256(out + 13, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 29) ,_mm256_slli_epi32( w1 , 3 ) ) ) ); - _mm256_storeu_si256(out + 14, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 14) ) ); - w0 = _mm256_lddqu_si256 (compressed + 8); - _mm256_storeu_si256(out + 15, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 31) ,_mm256_slli_epi32( w0 , 1 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 9); - _mm256_storeu_si256(out + 16, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 16) ,_mm256_slli_epi32( w1 , 16 ) ) ) ); - _mm256_storeu_si256(out + 17, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 1) ) ); - w0 = _mm256_lddqu_si256 (compressed + 10); - _mm256_storeu_si256(out + 18, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 18) ,_mm256_slli_epi32( w0 , 14 ) ) ) ); - _mm256_storeu_si256(out + 19, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 3) ) ); - w1 = _mm256_lddqu_si256 (compressed + 11); - _mm256_storeu_si256(out + 20, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 20) ,_mm256_slli_epi32( w1 , 12 ) ) ) ); - _mm256_storeu_si256(out + 21, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 5) ) ); - w0 = _mm256_lddqu_si256 (compressed + 12); - _mm256_storeu_si256(out + 22, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 22) ,_mm256_slli_epi32( w0 , 10 ) ) ) ); - _mm256_storeu_si256(out + 23, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 7) ) ); - w1 = _mm256_lddqu_si256 (compressed + 13); - _mm256_storeu_si256(out + 24, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 24) ,_mm256_slli_epi32( w1 , 8 ) ) ) ); - _mm256_storeu_si256(out + 25, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 9) ) ); - w0 = _mm256_lddqu_si256 (compressed + 14); - _mm256_storeu_si256(out + 26, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 26) ,_mm256_slli_epi32( w0 , 6 ) ) ) ); - _mm256_storeu_si256(out + 27, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 11) ) ); - w1 = _mm256_lddqu_si256 (compressed + 15); - _mm256_storeu_si256(out + 28, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 28) ,_mm256_slli_epi32( w1 , 4 ) ) ) ); - _mm256_storeu_si256(out + 29, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 13) ) ); - w0 = _mm256_lddqu_si256 (compressed + 16); - _mm256_storeu_si256(out + 30, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 30) ,_mm256_slli_epi32( w0 , 2 ) ) ) ); - _mm256_storeu_si256(out + 31, _mm256_srli_epi32( w0 , 15) ); -} - - -/* we packed 256 18-bit values, touching 18 256-bit words, using 288 bytes */ -static void avxunpackblock18(const __m256i * compressed, uint32_t * pout) { - /* we are going to access 18 256-bit words */ - __m256i w0, w1; - __m256i * out = (__m256i *) pout; - const __m256i mask = _mm256_set1_epi32(262143); - w0 = _mm256_lddqu_si256 (compressed); - _mm256_storeu_si256(out + 0, _mm256_and_si256 ( mask, w0 ) ); - w1 = _mm256_lddqu_si256 (compressed + 1); - _mm256_storeu_si256(out + 1, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 18) ,_mm256_slli_epi32( w1 , 14 ) ) ) ); - _mm256_storeu_si256(out + 2, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 4) ) ); - w0 = _mm256_lddqu_si256 (compressed + 2); - _mm256_storeu_si256(out + 3, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 22) ,_mm256_slli_epi32( w0 , 10 ) ) ) ); - _mm256_storeu_si256(out + 4, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 8) ) ); - w1 = _mm256_lddqu_si256 (compressed + 3); - _mm256_storeu_si256(out + 5, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 26) ,_mm256_slli_epi32( w1 , 6 ) ) ) ); - _mm256_storeu_si256(out + 6, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 12) ) ); - w0 = _mm256_lddqu_si256 (compressed + 4); - _mm256_storeu_si256(out + 7, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 30) ,_mm256_slli_epi32( w0 , 2 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 5); - _mm256_storeu_si256(out + 8, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 16) ,_mm256_slli_epi32( w1 , 16 ) ) ) ); - _mm256_storeu_si256(out + 9, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 2) ) ); - w0 = _mm256_lddqu_si256 (compressed + 6); - _mm256_storeu_si256(out + 10, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 20) ,_mm256_slli_epi32( w0 , 12 ) ) ) ); - _mm256_storeu_si256(out + 11, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 6) ) ); - w1 = _mm256_lddqu_si256 (compressed + 7); - _mm256_storeu_si256(out + 12, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 24) ,_mm256_slli_epi32( w1 , 8 ) ) ) ); - _mm256_storeu_si256(out + 13, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 10) ) ); - w0 = _mm256_lddqu_si256 (compressed + 8); - _mm256_storeu_si256(out + 14, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 28) ,_mm256_slli_epi32( w0 , 4 ) ) ) ); - _mm256_storeu_si256(out + 15, _mm256_srli_epi32( w0 , 14) ); - w1 = _mm256_lddqu_si256 (compressed + 9); - _mm256_storeu_si256(out + 16, _mm256_and_si256 ( mask, w1 ) ); - w0 = _mm256_lddqu_si256 (compressed + 10); - _mm256_storeu_si256(out + 17, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 18) ,_mm256_slli_epi32( w0 , 14 ) ) ) ); - _mm256_storeu_si256(out + 18, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 4) ) ); - w1 = _mm256_lddqu_si256 (compressed + 11); - _mm256_storeu_si256(out + 19, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 22) ,_mm256_slli_epi32( w1 , 10 ) ) ) ); - _mm256_storeu_si256(out + 20, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 8) ) ); - w0 = _mm256_lddqu_si256 (compressed + 12); - _mm256_storeu_si256(out + 21, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 26) ,_mm256_slli_epi32( w0 , 6 ) ) ) ); - _mm256_storeu_si256(out + 22, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 12) ) ); - w1 = _mm256_lddqu_si256 (compressed + 13); - _mm256_storeu_si256(out + 23, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 30) ,_mm256_slli_epi32( w1 , 2 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 14); - _mm256_storeu_si256(out + 24, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 16) ,_mm256_slli_epi32( w0 , 16 ) ) ) ); - _mm256_storeu_si256(out + 25, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 2) ) ); - w1 = _mm256_lddqu_si256 (compressed + 15); - _mm256_storeu_si256(out + 26, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 20) ,_mm256_slli_epi32( w1 , 12 ) ) ) ); - _mm256_storeu_si256(out + 27, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 6) ) ); - w0 = _mm256_lddqu_si256 (compressed + 16); - _mm256_storeu_si256(out + 28, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 24) ,_mm256_slli_epi32( w0 , 8 ) ) ) ); - _mm256_storeu_si256(out + 29, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 10) ) ); - w1 = _mm256_lddqu_si256 (compressed + 17); - _mm256_storeu_si256(out + 30, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 28) ,_mm256_slli_epi32( w1 , 4 ) ) ) ); - _mm256_storeu_si256(out + 31, _mm256_srli_epi32( w1 , 14) ); -} - - -/* we packed 256 19-bit values, touching 19 256-bit words, using 304 bytes */ -static void avxunpackblock19(const __m256i * compressed, uint32_t * pout) { - /* we are going to access 19 256-bit words */ - __m256i w0, w1; - __m256i * out = (__m256i *) pout; - const __m256i mask = _mm256_set1_epi32(524287); - w0 = _mm256_lddqu_si256 (compressed); - _mm256_storeu_si256(out + 0, _mm256_and_si256 ( mask, w0 ) ); - w1 = _mm256_lddqu_si256 (compressed + 1); - _mm256_storeu_si256(out + 1, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 19) ,_mm256_slli_epi32( w1 , 13 ) ) ) ); - _mm256_storeu_si256(out + 2, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 6) ) ); - w0 = _mm256_lddqu_si256 (compressed + 2); - _mm256_storeu_si256(out + 3, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 25) ,_mm256_slli_epi32( w0 , 7 ) ) ) ); - _mm256_storeu_si256(out + 4, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 12) ) ); - w1 = _mm256_lddqu_si256 (compressed + 3); - _mm256_storeu_si256(out + 5, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 31) ,_mm256_slli_epi32( w1 , 1 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 4); - _mm256_storeu_si256(out + 6, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 18) ,_mm256_slli_epi32( w0 , 14 ) ) ) ); - _mm256_storeu_si256(out + 7, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 5) ) ); - w1 = _mm256_lddqu_si256 (compressed + 5); - _mm256_storeu_si256(out + 8, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 24) ,_mm256_slli_epi32( w1 , 8 ) ) ) ); - _mm256_storeu_si256(out + 9, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 11) ) ); - w0 = _mm256_lddqu_si256 (compressed + 6); - _mm256_storeu_si256(out + 10, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 30) ,_mm256_slli_epi32( w0 , 2 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 7); - _mm256_storeu_si256(out + 11, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 17) ,_mm256_slli_epi32( w1 , 15 ) ) ) ); - _mm256_storeu_si256(out + 12, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 4) ) ); - w0 = _mm256_lddqu_si256 (compressed + 8); - _mm256_storeu_si256(out + 13, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 23) ,_mm256_slli_epi32( w0 , 9 ) ) ) ); - _mm256_storeu_si256(out + 14, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 10) ) ); - w1 = _mm256_lddqu_si256 (compressed + 9); - _mm256_storeu_si256(out + 15, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 29) ,_mm256_slli_epi32( w1 , 3 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 10); - _mm256_storeu_si256(out + 16, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 16) ,_mm256_slli_epi32( w0 , 16 ) ) ) ); - _mm256_storeu_si256(out + 17, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 3) ) ); - w1 = _mm256_lddqu_si256 (compressed + 11); - _mm256_storeu_si256(out + 18, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 22) ,_mm256_slli_epi32( w1 , 10 ) ) ) ); - _mm256_storeu_si256(out + 19, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 9) ) ); - w0 = _mm256_lddqu_si256 (compressed + 12); - _mm256_storeu_si256(out + 20, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 28) ,_mm256_slli_epi32( w0 , 4 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 13); - _mm256_storeu_si256(out + 21, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 15) ,_mm256_slli_epi32( w1 , 17 ) ) ) ); - _mm256_storeu_si256(out + 22, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 2) ) ); - w0 = _mm256_lddqu_si256 (compressed + 14); - _mm256_storeu_si256(out + 23, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 21) ,_mm256_slli_epi32( w0 , 11 ) ) ) ); - _mm256_storeu_si256(out + 24, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 8) ) ); - w1 = _mm256_lddqu_si256 (compressed + 15); - _mm256_storeu_si256(out + 25, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 27) ,_mm256_slli_epi32( w1 , 5 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 16); - _mm256_storeu_si256(out + 26, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 14) ,_mm256_slli_epi32( w0 , 18 ) ) ) ); - _mm256_storeu_si256(out + 27, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 1) ) ); - w1 = _mm256_lddqu_si256 (compressed + 17); - _mm256_storeu_si256(out + 28, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 20) ,_mm256_slli_epi32( w1 , 12 ) ) ) ); - _mm256_storeu_si256(out + 29, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 7) ) ); - w0 = _mm256_lddqu_si256 (compressed + 18); - _mm256_storeu_si256(out + 30, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 26) ,_mm256_slli_epi32( w0 , 6 ) ) ) ); - _mm256_storeu_si256(out + 31, _mm256_srli_epi32( w0 , 13) ); -} - - -/* we packed 256 20-bit values, touching 20 256-bit words, using 320 bytes */ -static void avxunpackblock20(const __m256i * compressed, uint32_t * pout) { - /* we are going to access 20 256-bit words */ - __m256i w0, w1; - __m256i * out = (__m256i *) pout; - const __m256i mask = _mm256_set1_epi32(1048575); - w0 = _mm256_lddqu_si256 (compressed); - _mm256_storeu_si256(out + 0, _mm256_and_si256 ( mask, w0 ) ); - w1 = _mm256_lddqu_si256 (compressed + 1); - _mm256_storeu_si256(out + 1, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 20) ,_mm256_slli_epi32( w1 , 12 ) ) ) ); - _mm256_storeu_si256(out + 2, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 8) ) ); - w0 = _mm256_lddqu_si256 (compressed + 2); - _mm256_storeu_si256(out + 3, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 28) ,_mm256_slli_epi32( w0 , 4 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 3); - _mm256_storeu_si256(out + 4, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 16) ,_mm256_slli_epi32( w1 , 16 ) ) ) ); - _mm256_storeu_si256(out + 5, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 4) ) ); - w0 = _mm256_lddqu_si256 (compressed + 4); - _mm256_storeu_si256(out + 6, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 24) ,_mm256_slli_epi32( w0 , 8 ) ) ) ); - _mm256_storeu_si256(out + 7, _mm256_srli_epi32( w0 , 12) ); - w1 = _mm256_lddqu_si256 (compressed + 5); - _mm256_storeu_si256(out + 8, _mm256_and_si256 ( mask, w1 ) ); - w0 = _mm256_lddqu_si256 (compressed + 6); - _mm256_storeu_si256(out + 9, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 20) ,_mm256_slli_epi32( w0 , 12 ) ) ) ); - _mm256_storeu_si256(out + 10, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 8) ) ); - w1 = _mm256_lddqu_si256 (compressed + 7); - _mm256_storeu_si256(out + 11, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 28) ,_mm256_slli_epi32( w1 , 4 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 8); - _mm256_storeu_si256(out + 12, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 16) ,_mm256_slli_epi32( w0 , 16 ) ) ) ); - _mm256_storeu_si256(out + 13, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 4) ) ); - w1 = _mm256_lddqu_si256 (compressed + 9); - _mm256_storeu_si256(out + 14, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 24) ,_mm256_slli_epi32( w1 , 8 ) ) ) ); - _mm256_storeu_si256(out + 15, _mm256_srli_epi32( w1 , 12) ); - w0 = _mm256_lddqu_si256 (compressed + 10); - _mm256_storeu_si256(out + 16, _mm256_and_si256 ( mask, w0 ) ); - w1 = _mm256_lddqu_si256 (compressed + 11); - _mm256_storeu_si256(out + 17, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 20) ,_mm256_slli_epi32( w1 , 12 ) ) ) ); - _mm256_storeu_si256(out + 18, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 8) ) ); - w0 = _mm256_lddqu_si256 (compressed + 12); - _mm256_storeu_si256(out + 19, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 28) ,_mm256_slli_epi32( w0 , 4 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 13); - _mm256_storeu_si256(out + 20, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 16) ,_mm256_slli_epi32( w1 , 16 ) ) ) ); - _mm256_storeu_si256(out + 21, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 4) ) ); - w0 = _mm256_lddqu_si256 (compressed + 14); - _mm256_storeu_si256(out + 22, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 24) ,_mm256_slli_epi32( w0 , 8 ) ) ) ); - _mm256_storeu_si256(out + 23, _mm256_srli_epi32( w0 , 12) ); - w1 = _mm256_lddqu_si256 (compressed + 15); - _mm256_storeu_si256(out + 24, _mm256_and_si256 ( mask, w1 ) ); - w0 = _mm256_lddqu_si256 (compressed + 16); - _mm256_storeu_si256(out + 25, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 20) ,_mm256_slli_epi32( w0 , 12 ) ) ) ); - _mm256_storeu_si256(out + 26, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 8) ) ); - w1 = _mm256_lddqu_si256 (compressed + 17); - _mm256_storeu_si256(out + 27, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 28) ,_mm256_slli_epi32( w1 , 4 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 18); - _mm256_storeu_si256(out + 28, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 16) ,_mm256_slli_epi32( w0 , 16 ) ) ) ); - _mm256_storeu_si256(out + 29, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 4) ) ); - w1 = _mm256_lddqu_si256 (compressed + 19); - _mm256_storeu_si256(out + 30, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 24) ,_mm256_slli_epi32( w1 , 8 ) ) ) ); - _mm256_storeu_si256(out + 31, _mm256_srli_epi32( w1 , 12) ); -} - - -/* we packed 256 21-bit values, touching 21 256-bit words, using 336 bytes */ -static void avxunpackblock21(const __m256i * compressed, uint32_t * pout) { - /* we are going to access 21 256-bit words */ - __m256i w0, w1; - __m256i * out = (__m256i *) pout; - const __m256i mask = _mm256_set1_epi32(2097151); - w0 = _mm256_lddqu_si256 (compressed); - _mm256_storeu_si256(out + 0, _mm256_and_si256 ( mask, w0 ) ); - w1 = _mm256_lddqu_si256 (compressed + 1); - _mm256_storeu_si256(out + 1, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 21) ,_mm256_slli_epi32( w1 , 11 ) ) ) ); - _mm256_storeu_si256(out + 2, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 10) ) ); - w0 = _mm256_lddqu_si256 (compressed + 2); - _mm256_storeu_si256(out + 3, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 31) ,_mm256_slli_epi32( w0 , 1 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 3); - _mm256_storeu_si256(out + 4, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 20) ,_mm256_slli_epi32( w1 , 12 ) ) ) ); - _mm256_storeu_si256(out + 5, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 9) ) ); - w0 = _mm256_lddqu_si256 (compressed + 4); - _mm256_storeu_si256(out + 6, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 30) ,_mm256_slli_epi32( w0 , 2 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 5); - _mm256_storeu_si256(out + 7, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 19) ,_mm256_slli_epi32( w1 , 13 ) ) ) ); - _mm256_storeu_si256(out + 8, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 8) ) ); - w0 = _mm256_lddqu_si256 (compressed + 6); - _mm256_storeu_si256(out + 9, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 29) ,_mm256_slli_epi32( w0 , 3 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 7); - _mm256_storeu_si256(out + 10, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 18) ,_mm256_slli_epi32( w1 , 14 ) ) ) ); - _mm256_storeu_si256(out + 11, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 7) ) ); - w0 = _mm256_lddqu_si256 (compressed + 8); - _mm256_storeu_si256(out + 12, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 28) ,_mm256_slli_epi32( w0 , 4 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 9); - _mm256_storeu_si256(out + 13, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 17) ,_mm256_slli_epi32( w1 , 15 ) ) ) ); - _mm256_storeu_si256(out + 14, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 6) ) ); - w0 = _mm256_lddqu_si256 (compressed + 10); - _mm256_storeu_si256(out + 15, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 27) ,_mm256_slli_epi32( w0 , 5 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 11); - _mm256_storeu_si256(out + 16, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 16) ,_mm256_slli_epi32( w1 , 16 ) ) ) ); - _mm256_storeu_si256(out + 17, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 5) ) ); - w0 = _mm256_lddqu_si256 (compressed + 12); - _mm256_storeu_si256(out + 18, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 26) ,_mm256_slli_epi32( w0 , 6 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 13); - _mm256_storeu_si256(out + 19, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 15) ,_mm256_slli_epi32( w1 , 17 ) ) ) ); - _mm256_storeu_si256(out + 20, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 4) ) ); - w0 = _mm256_lddqu_si256 (compressed + 14); - _mm256_storeu_si256(out + 21, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 25) ,_mm256_slli_epi32( w0 , 7 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 15); - _mm256_storeu_si256(out + 22, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 14) ,_mm256_slli_epi32( w1 , 18 ) ) ) ); - _mm256_storeu_si256(out + 23, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 3) ) ); - w0 = _mm256_lddqu_si256 (compressed + 16); - _mm256_storeu_si256(out + 24, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 24) ,_mm256_slli_epi32( w0 , 8 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 17); - _mm256_storeu_si256(out + 25, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 13) ,_mm256_slli_epi32( w1 , 19 ) ) ) ); - _mm256_storeu_si256(out + 26, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 2) ) ); - w0 = _mm256_lddqu_si256 (compressed + 18); - _mm256_storeu_si256(out + 27, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 23) ,_mm256_slli_epi32( w0 , 9 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 19); - _mm256_storeu_si256(out + 28, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 12) ,_mm256_slli_epi32( w1 , 20 ) ) ) ); - _mm256_storeu_si256(out + 29, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 1) ) ); - w0 = _mm256_lddqu_si256 (compressed + 20); - _mm256_storeu_si256(out + 30, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 22) ,_mm256_slli_epi32( w0 , 10 ) ) ) ); - _mm256_storeu_si256(out + 31, _mm256_srli_epi32( w0 , 11) ); -} - - -/* we packed 256 22-bit values, touching 22 256-bit words, using 352 bytes */ -static void avxunpackblock22(const __m256i * compressed, uint32_t * pout) { - /* we are going to access 22 256-bit words */ - __m256i w0, w1; - __m256i * out = (__m256i *) pout; - const __m256i mask = _mm256_set1_epi32(4194303); - w0 = _mm256_lddqu_si256 (compressed); - _mm256_storeu_si256(out + 0, _mm256_and_si256 ( mask, w0 ) ); - w1 = _mm256_lddqu_si256 (compressed + 1); - _mm256_storeu_si256(out + 1, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 22) ,_mm256_slli_epi32( w1 , 10 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 2); - _mm256_storeu_si256(out + 2, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 12) ,_mm256_slli_epi32( w0 , 20 ) ) ) ); - _mm256_storeu_si256(out + 3, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 2) ) ); - w1 = _mm256_lddqu_si256 (compressed + 3); - _mm256_storeu_si256(out + 4, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 24) ,_mm256_slli_epi32( w1 , 8 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 4); - _mm256_storeu_si256(out + 5, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 14) ,_mm256_slli_epi32( w0 , 18 ) ) ) ); - _mm256_storeu_si256(out + 6, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 4) ) ); - w1 = _mm256_lddqu_si256 (compressed + 5); - _mm256_storeu_si256(out + 7, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 26) ,_mm256_slli_epi32( w1 , 6 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 6); - _mm256_storeu_si256(out + 8, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 16) ,_mm256_slli_epi32( w0 , 16 ) ) ) ); - _mm256_storeu_si256(out + 9, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 6) ) ); - w1 = _mm256_lddqu_si256 (compressed + 7); - _mm256_storeu_si256(out + 10, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 28) ,_mm256_slli_epi32( w1 , 4 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 8); - _mm256_storeu_si256(out + 11, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 18) ,_mm256_slli_epi32( w0 , 14 ) ) ) ); - _mm256_storeu_si256(out + 12, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 8) ) ); - w1 = _mm256_lddqu_si256 (compressed + 9); - _mm256_storeu_si256(out + 13, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 30) ,_mm256_slli_epi32( w1 , 2 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 10); - _mm256_storeu_si256(out + 14, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 20) ,_mm256_slli_epi32( w0 , 12 ) ) ) ); - _mm256_storeu_si256(out + 15, _mm256_srli_epi32( w0 , 10) ); - w1 = _mm256_lddqu_si256 (compressed + 11); - _mm256_storeu_si256(out + 16, _mm256_and_si256 ( mask, w1 ) ); - w0 = _mm256_lddqu_si256 (compressed + 12); - _mm256_storeu_si256(out + 17, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 22) ,_mm256_slli_epi32( w0 , 10 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 13); - _mm256_storeu_si256(out + 18, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 12) ,_mm256_slli_epi32( w1 , 20 ) ) ) ); - _mm256_storeu_si256(out + 19, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 2) ) ); - w0 = _mm256_lddqu_si256 (compressed + 14); - _mm256_storeu_si256(out + 20, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 24) ,_mm256_slli_epi32( w0 , 8 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 15); - _mm256_storeu_si256(out + 21, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 14) ,_mm256_slli_epi32( w1 , 18 ) ) ) ); - _mm256_storeu_si256(out + 22, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 4) ) ); - w0 = _mm256_lddqu_si256 (compressed + 16); - _mm256_storeu_si256(out + 23, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 26) ,_mm256_slli_epi32( w0 , 6 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 17); - _mm256_storeu_si256(out + 24, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 16) ,_mm256_slli_epi32( w1 , 16 ) ) ) ); - _mm256_storeu_si256(out + 25, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 6) ) ); - w0 = _mm256_lddqu_si256 (compressed + 18); - _mm256_storeu_si256(out + 26, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 28) ,_mm256_slli_epi32( w0 , 4 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 19); - _mm256_storeu_si256(out + 27, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 18) ,_mm256_slli_epi32( w1 , 14 ) ) ) ); - _mm256_storeu_si256(out + 28, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 8) ) ); - w0 = _mm256_lddqu_si256 (compressed + 20); - _mm256_storeu_si256(out + 29, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 30) ,_mm256_slli_epi32( w0 , 2 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 21); - _mm256_storeu_si256(out + 30, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 20) ,_mm256_slli_epi32( w1 , 12 ) ) ) ); - _mm256_storeu_si256(out + 31, _mm256_srli_epi32( w1 , 10) ); -} - - -/* we packed 256 23-bit values, touching 23 256-bit words, using 368 bytes */ -static void avxunpackblock23(const __m256i * compressed, uint32_t * pout) { - /* we are going to access 23 256-bit words */ - __m256i w0, w1; - __m256i * out = (__m256i *) pout; - const __m256i mask = _mm256_set1_epi32(8388607); - w0 = _mm256_lddqu_si256 (compressed); - _mm256_storeu_si256(out + 0, _mm256_and_si256 ( mask, w0 ) ); - w1 = _mm256_lddqu_si256 (compressed + 1); - _mm256_storeu_si256(out + 1, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 23) ,_mm256_slli_epi32( w1 , 9 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 2); - _mm256_storeu_si256(out + 2, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 14) ,_mm256_slli_epi32( w0 , 18 ) ) ) ); - _mm256_storeu_si256(out + 3, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 5) ) ); - w1 = _mm256_lddqu_si256 (compressed + 3); - _mm256_storeu_si256(out + 4, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 28) ,_mm256_slli_epi32( w1 , 4 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 4); - _mm256_storeu_si256(out + 5, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 19) ,_mm256_slli_epi32( w0 , 13 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 5); - _mm256_storeu_si256(out + 6, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 10) ,_mm256_slli_epi32( w1 , 22 ) ) ) ); - _mm256_storeu_si256(out + 7, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 1) ) ); - w0 = _mm256_lddqu_si256 (compressed + 6); - _mm256_storeu_si256(out + 8, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 24) ,_mm256_slli_epi32( w0 , 8 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 7); - _mm256_storeu_si256(out + 9, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 15) ,_mm256_slli_epi32( w1 , 17 ) ) ) ); - _mm256_storeu_si256(out + 10, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 6) ) ); - w0 = _mm256_lddqu_si256 (compressed + 8); - _mm256_storeu_si256(out + 11, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 29) ,_mm256_slli_epi32( w0 , 3 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 9); - _mm256_storeu_si256(out + 12, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 20) ,_mm256_slli_epi32( w1 , 12 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 10); - _mm256_storeu_si256(out + 13, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 11) ,_mm256_slli_epi32( w0 , 21 ) ) ) ); - _mm256_storeu_si256(out + 14, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 2) ) ); - w1 = _mm256_lddqu_si256 (compressed + 11); - _mm256_storeu_si256(out + 15, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 25) ,_mm256_slli_epi32( w1 , 7 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 12); - _mm256_storeu_si256(out + 16, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 16) ,_mm256_slli_epi32( w0 , 16 ) ) ) ); - _mm256_storeu_si256(out + 17, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 7) ) ); - w1 = _mm256_lddqu_si256 (compressed + 13); - _mm256_storeu_si256(out + 18, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 30) ,_mm256_slli_epi32( w1 , 2 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 14); - _mm256_storeu_si256(out + 19, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 21) ,_mm256_slli_epi32( w0 , 11 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 15); - _mm256_storeu_si256(out + 20, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 12) ,_mm256_slli_epi32( w1 , 20 ) ) ) ); - _mm256_storeu_si256(out + 21, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 3) ) ); - w0 = _mm256_lddqu_si256 (compressed + 16); - _mm256_storeu_si256(out + 22, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 26) ,_mm256_slli_epi32( w0 , 6 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 17); - _mm256_storeu_si256(out + 23, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 17) ,_mm256_slli_epi32( w1 , 15 ) ) ) ); - _mm256_storeu_si256(out + 24, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 8) ) ); - w0 = _mm256_lddqu_si256 (compressed + 18); - _mm256_storeu_si256(out + 25, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 31) ,_mm256_slli_epi32( w0 , 1 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 19); - _mm256_storeu_si256(out + 26, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 22) ,_mm256_slli_epi32( w1 , 10 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 20); - _mm256_storeu_si256(out + 27, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 13) ,_mm256_slli_epi32( w0 , 19 ) ) ) ); - _mm256_storeu_si256(out + 28, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 4) ) ); - w1 = _mm256_lddqu_si256 (compressed + 21); - _mm256_storeu_si256(out + 29, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 27) ,_mm256_slli_epi32( w1 , 5 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 22); - _mm256_storeu_si256(out + 30, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 18) ,_mm256_slli_epi32( w0 , 14 ) ) ) ); - _mm256_storeu_si256(out + 31, _mm256_srli_epi32( w0 , 9) ); -} - - -/* we packed 256 24-bit values, touching 24 256-bit words, using 384 bytes */ -static void avxunpackblock24(const __m256i * compressed, uint32_t * pout) { - /* we are going to access 24 256-bit words */ - __m256i w0, w1; - __m256i * out = (__m256i *) pout; - const __m256i mask = _mm256_set1_epi32(16777215); - w0 = _mm256_lddqu_si256 (compressed); - _mm256_storeu_si256(out + 0, _mm256_and_si256 ( mask, w0 ) ); - w1 = _mm256_lddqu_si256 (compressed + 1); - _mm256_storeu_si256(out + 1, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 24) ,_mm256_slli_epi32( w1 , 8 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 2); - _mm256_storeu_si256(out + 2, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 16) ,_mm256_slli_epi32( w0 , 16 ) ) ) ); - _mm256_storeu_si256(out + 3, _mm256_srli_epi32( w0 , 8) ); - w1 = _mm256_lddqu_si256 (compressed + 3); - _mm256_storeu_si256(out + 4, _mm256_and_si256 ( mask, w1 ) ); - w0 = _mm256_lddqu_si256 (compressed + 4); - _mm256_storeu_si256(out + 5, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 24) ,_mm256_slli_epi32( w0 , 8 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 5); - _mm256_storeu_si256(out + 6, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 16) ,_mm256_slli_epi32( w1 , 16 ) ) ) ); - _mm256_storeu_si256(out + 7, _mm256_srli_epi32( w1 , 8) ); - w0 = _mm256_lddqu_si256 (compressed + 6); - _mm256_storeu_si256(out + 8, _mm256_and_si256 ( mask, w0 ) ); - w1 = _mm256_lddqu_si256 (compressed + 7); - _mm256_storeu_si256(out + 9, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 24) ,_mm256_slli_epi32( w1 , 8 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 8); - _mm256_storeu_si256(out + 10, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 16) ,_mm256_slli_epi32( w0 , 16 ) ) ) ); - _mm256_storeu_si256(out + 11, _mm256_srli_epi32( w0 , 8) ); - w1 = _mm256_lddqu_si256 (compressed + 9); - _mm256_storeu_si256(out + 12, _mm256_and_si256 ( mask, w1 ) ); - w0 = _mm256_lddqu_si256 (compressed + 10); - _mm256_storeu_si256(out + 13, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 24) ,_mm256_slli_epi32( w0 , 8 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 11); - _mm256_storeu_si256(out + 14, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 16) ,_mm256_slli_epi32( w1 , 16 ) ) ) ); - _mm256_storeu_si256(out + 15, _mm256_srli_epi32( w1 , 8) ); - w0 = _mm256_lddqu_si256 (compressed + 12); - _mm256_storeu_si256(out + 16, _mm256_and_si256 ( mask, w0 ) ); - w1 = _mm256_lddqu_si256 (compressed + 13); - _mm256_storeu_si256(out + 17, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 24) ,_mm256_slli_epi32( w1 , 8 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 14); - _mm256_storeu_si256(out + 18, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 16) ,_mm256_slli_epi32( w0 , 16 ) ) ) ); - _mm256_storeu_si256(out + 19, _mm256_srli_epi32( w0 , 8) ); - w1 = _mm256_lddqu_si256 (compressed + 15); - _mm256_storeu_si256(out + 20, _mm256_and_si256 ( mask, w1 ) ); - w0 = _mm256_lddqu_si256 (compressed + 16); - _mm256_storeu_si256(out + 21, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 24) ,_mm256_slli_epi32( w0 , 8 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 17); - _mm256_storeu_si256(out + 22, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 16) ,_mm256_slli_epi32( w1 , 16 ) ) ) ); - _mm256_storeu_si256(out + 23, _mm256_srli_epi32( w1 , 8) ); - w0 = _mm256_lddqu_si256 (compressed + 18); - _mm256_storeu_si256(out + 24, _mm256_and_si256 ( mask, w0 ) ); - w1 = _mm256_lddqu_si256 (compressed + 19); - _mm256_storeu_si256(out + 25, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 24) ,_mm256_slli_epi32( w1 , 8 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 20); - _mm256_storeu_si256(out + 26, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 16) ,_mm256_slli_epi32( w0 , 16 ) ) ) ); - _mm256_storeu_si256(out + 27, _mm256_srli_epi32( w0 , 8) ); - w1 = _mm256_lddqu_si256 (compressed + 21); - _mm256_storeu_si256(out + 28, _mm256_and_si256 ( mask, w1 ) ); - w0 = _mm256_lddqu_si256 (compressed + 22); - _mm256_storeu_si256(out + 29, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 24) ,_mm256_slli_epi32( w0 , 8 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 23); - _mm256_storeu_si256(out + 30, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 16) ,_mm256_slli_epi32( w1 , 16 ) ) ) ); - _mm256_storeu_si256(out + 31, _mm256_srli_epi32( w1 , 8) ); -} - - -/* we packed 256 25-bit values, touching 25 256-bit words, using 400 bytes */ -static void avxunpackblock25(const __m256i * compressed, uint32_t * pout) { - /* we are going to access 25 256-bit words */ - __m256i w0, w1; - __m256i * out = (__m256i *) pout; - const __m256i mask = _mm256_set1_epi32(33554431); - w0 = _mm256_lddqu_si256 (compressed); - _mm256_storeu_si256(out + 0, _mm256_and_si256 ( mask, w0 ) ); - w1 = _mm256_lddqu_si256 (compressed + 1); - _mm256_storeu_si256(out + 1, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 25) ,_mm256_slli_epi32( w1 , 7 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 2); - _mm256_storeu_si256(out + 2, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 18) ,_mm256_slli_epi32( w0 , 14 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 3); - _mm256_storeu_si256(out + 3, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 11) ,_mm256_slli_epi32( w1 , 21 ) ) ) ); - _mm256_storeu_si256(out + 4, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 4) ) ); - w0 = _mm256_lddqu_si256 (compressed + 4); - _mm256_storeu_si256(out + 5, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 29) ,_mm256_slli_epi32( w0 , 3 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 5); - _mm256_storeu_si256(out + 6, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 22) ,_mm256_slli_epi32( w1 , 10 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 6); - _mm256_storeu_si256(out + 7, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 15) ,_mm256_slli_epi32( w0 , 17 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 7); - _mm256_storeu_si256(out + 8, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 8) ,_mm256_slli_epi32( w1 , 24 ) ) ) ); - _mm256_storeu_si256(out + 9, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 1) ) ); - w0 = _mm256_lddqu_si256 (compressed + 8); - _mm256_storeu_si256(out + 10, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 26) ,_mm256_slli_epi32( w0 , 6 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 9); - _mm256_storeu_si256(out + 11, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 19) ,_mm256_slli_epi32( w1 , 13 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 10); - _mm256_storeu_si256(out + 12, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 12) ,_mm256_slli_epi32( w0 , 20 ) ) ) ); - _mm256_storeu_si256(out + 13, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 5) ) ); - w1 = _mm256_lddqu_si256 (compressed + 11); - _mm256_storeu_si256(out + 14, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 30) ,_mm256_slli_epi32( w1 , 2 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 12); - _mm256_storeu_si256(out + 15, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 23) ,_mm256_slli_epi32( w0 , 9 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 13); - _mm256_storeu_si256(out + 16, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 16) ,_mm256_slli_epi32( w1 , 16 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 14); - _mm256_storeu_si256(out + 17, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 9) ,_mm256_slli_epi32( w0 , 23 ) ) ) ); - _mm256_storeu_si256(out + 18, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 2) ) ); - w1 = _mm256_lddqu_si256 (compressed + 15); - _mm256_storeu_si256(out + 19, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 27) ,_mm256_slli_epi32( w1 , 5 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 16); - _mm256_storeu_si256(out + 20, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 20) ,_mm256_slli_epi32( w0 , 12 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 17); - _mm256_storeu_si256(out + 21, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 13) ,_mm256_slli_epi32( w1 , 19 ) ) ) ); - _mm256_storeu_si256(out + 22, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 6) ) ); - w0 = _mm256_lddqu_si256 (compressed + 18); - _mm256_storeu_si256(out + 23, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 31) ,_mm256_slli_epi32( w0 , 1 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 19); - _mm256_storeu_si256(out + 24, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 24) ,_mm256_slli_epi32( w1 , 8 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 20); - _mm256_storeu_si256(out + 25, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 17) ,_mm256_slli_epi32( w0 , 15 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 21); - _mm256_storeu_si256(out + 26, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 10) ,_mm256_slli_epi32( w1 , 22 ) ) ) ); - _mm256_storeu_si256(out + 27, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 3) ) ); - w0 = _mm256_lddqu_si256 (compressed + 22); - _mm256_storeu_si256(out + 28, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 28) ,_mm256_slli_epi32( w0 , 4 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 23); - _mm256_storeu_si256(out + 29, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 21) ,_mm256_slli_epi32( w1 , 11 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 24); - _mm256_storeu_si256(out + 30, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 14) ,_mm256_slli_epi32( w0 , 18 ) ) ) ); - _mm256_storeu_si256(out + 31, _mm256_srli_epi32( w0 , 7) ); -} - - -/* we packed 256 26-bit values, touching 26 256-bit words, using 416 bytes */ -static void avxunpackblock26(const __m256i * compressed, uint32_t * pout) { - /* we are going to access 26 256-bit words */ - __m256i w0, w1; - __m256i * out = (__m256i *) pout; - const __m256i mask = _mm256_set1_epi32(67108863); - w0 = _mm256_lddqu_si256 (compressed); - _mm256_storeu_si256(out + 0, _mm256_and_si256 ( mask, w0 ) ); - w1 = _mm256_lddqu_si256 (compressed + 1); - _mm256_storeu_si256(out + 1, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 26) ,_mm256_slli_epi32( w1 , 6 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 2); - _mm256_storeu_si256(out + 2, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 20) ,_mm256_slli_epi32( w0 , 12 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 3); - _mm256_storeu_si256(out + 3, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 14) ,_mm256_slli_epi32( w1 , 18 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 4); - _mm256_storeu_si256(out + 4, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 8) ,_mm256_slli_epi32( w0 , 24 ) ) ) ); - _mm256_storeu_si256(out + 5, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 2) ) ); - w1 = _mm256_lddqu_si256 (compressed + 5); - _mm256_storeu_si256(out + 6, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 28) ,_mm256_slli_epi32( w1 , 4 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 6); - _mm256_storeu_si256(out + 7, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 22) ,_mm256_slli_epi32( w0 , 10 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 7); - _mm256_storeu_si256(out + 8, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 16) ,_mm256_slli_epi32( w1 , 16 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 8); - _mm256_storeu_si256(out + 9, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 10) ,_mm256_slli_epi32( w0 , 22 ) ) ) ); - _mm256_storeu_si256(out + 10, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 4) ) ); - w1 = _mm256_lddqu_si256 (compressed + 9); - _mm256_storeu_si256(out + 11, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 30) ,_mm256_slli_epi32( w1 , 2 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 10); - _mm256_storeu_si256(out + 12, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 24) ,_mm256_slli_epi32( w0 , 8 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 11); - _mm256_storeu_si256(out + 13, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 18) ,_mm256_slli_epi32( w1 , 14 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 12); - _mm256_storeu_si256(out + 14, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 12) ,_mm256_slli_epi32( w0 , 20 ) ) ) ); - _mm256_storeu_si256(out + 15, _mm256_srli_epi32( w0 , 6) ); - w1 = _mm256_lddqu_si256 (compressed + 13); - _mm256_storeu_si256(out + 16, _mm256_and_si256 ( mask, w1 ) ); - w0 = _mm256_lddqu_si256 (compressed + 14); - _mm256_storeu_si256(out + 17, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 26) ,_mm256_slli_epi32( w0 , 6 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 15); - _mm256_storeu_si256(out + 18, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 20) ,_mm256_slli_epi32( w1 , 12 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 16); - _mm256_storeu_si256(out + 19, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 14) ,_mm256_slli_epi32( w0 , 18 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 17); - _mm256_storeu_si256(out + 20, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 8) ,_mm256_slli_epi32( w1 , 24 ) ) ) ); - _mm256_storeu_si256(out + 21, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 2) ) ); - w0 = _mm256_lddqu_si256 (compressed + 18); - _mm256_storeu_si256(out + 22, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 28) ,_mm256_slli_epi32( w0 , 4 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 19); - _mm256_storeu_si256(out + 23, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 22) ,_mm256_slli_epi32( w1 , 10 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 20); - _mm256_storeu_si256(out + 24, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 16) ,_mm256_slli_epi32( w0 , 16 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 21); - _mm256_storeu_si256(out + 25, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 10) ,_mm256_slli_epi32( w1 , 22 ) ) ) ); - _mm256_storeu_si256(out + 26, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 4) ) ); - w0 = _mm256_lddqu_si256 (compressed + 22); - _mm256_storeu_si256(out + 27, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 30) ,_mm256_slli_epi32( w0 , 2 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 23); - _mm256_storeu_si256(out + 28, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 24) ,_mm256_slli_epi32( w1 , 8 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 24); - _mm256_storeu_si256(out + 29, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 18) ,_mm256_slli_epi32( w0 , 14 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 25); - _mm256_storeu_si256(out + 30, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 12) ,_mm256_slli_epi32( w1 , 20 ) ) ) ); - _mm256_storeu_si256(out + 31, _mm256_srli_epi32( w1 , 6) ); -} - - -/* we packed 256 27-bit values, touching 27 256-bit words, using 432 bytes */ -static void avxunpackblock27(const __m256i * compressed, uint32_t * pout) { - /* we are going to access 27 256-bit words */ - __m256i w0, w1; - __m256i * out = (__m256i *) pout; - const __m256i mask = _mm256_set1_epi32(134217727); - w0 = _mm256_lddqu_si256 (compressed); - _mm256_storeu_si256(out + 0, _mm256_and_si256 ( mask, w0 ) ); - w1 = _mm256_lddqu_si256 (compressed + 1); - _mm256_storeu_si256(out + 1, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 27) ,_mm256_slli_epi32( w1 , 5 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 2); - _mm256_storeu_si256(out + 2, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 22) ,_mm256_slli_epi32( w0 , 10 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 3); - _mm256_storeu_si256(out + 3, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 17) ,_mm256_slli_epi32( w1 , 15 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 4); - _mm256_storeu_si256(out + 4, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 12) ,_mm256_slli_epi32( w0 , 20 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 5); - _mm256_storeu_si256(out + 5, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 7) ,_mm256_slli_epi32( w1 , 25 ) ) ) ); - _mm256_storeu_si256(out + 6, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 2) ) ); - w0 = _mm256_lddqu_si256 (compressed + 6); - _mm256_storeu_si256(out + 7, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 29) ,_mm256_slli_epi32( w0 , 3 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 7); - _mm256_storeu_si256(out + 8, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 24) ,_mm256_slli_epi32( w1 , 8 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 8); - _mm256_storeu_si256(out + 9, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 19) ,_mm256_slli_epi32( w0 , 13 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 9); - _mm256_storeu_si256(out + 10, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 14) ,_mm256_slli_epi32( w1 , 18 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 10); - _mm256_storeu_si256(out + 11, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 9) ,_mm256_slli_epi32( w0 , 23 ) ) ) ); - _mm256_storeu_si256(out + 12, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 4) ) ); - w1 = _mm256_lddqu_si256 (compressed + 11); - _mm256_storeu_si256(out + 13, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 31) ,_mm256_slli_epi32( w1 , 1 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 12); - _mm256_storeu_si256(out + 14, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 26) ,_mm256_slli_epi32( w0 , 6 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 13); - _mm256_storeu_si256(out + 15, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 21) ,_mm256_slli_epi32( w1 , 11 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 14); - _mm256_storeu_si256(out + 16, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 16) ,_mm256_slli_epi32( w0 , 16 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 15); - _mm256_storeu_si256(out + 17, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 11) ,_mm256_slli_epi32( w1 , 21 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 16); - _mm256_storeu_si256(out + 18, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 6) ,_mm256_slli_epi32( w0 , 26 ) ) ) ); - _mm256_storeu_si256(out + 19, _mm256_and_si256 ( mask, _mm256_srli_epi32( w0 , 1) ) ); - w1 = _mm256_lddqu_si256 (compressed + 17); - _mm256_storeu_si256(out + 20, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 28) ,_mm256_slli_epi32( w1 , 4 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 18); - _mm256_storeu_si256(out + 21, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 23) ,_mm256_slli_epi32( w0 , 9 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 19); - _mm256_storeu_si256(out + 22, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 18) ,_mm256_slli_epi32( w1 , 14 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 20); - _mm256_storeu_si256(out + 23, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 13) ,_mm256_slli_epi32( w0 , 19 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 21); - _mm256_storeu_si256(out + 24, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 8) ,_mm256_slli_epi32( w1 , 24 ) ) ) ); - _mm256_storeu_si256(out + 25, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 3) ) ); - w0 = _mm256_lddqu_si256 (compressed + 22); - _mm256_storeu_si256(out + 26, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 30) ,_mm256_slli_epi32( w0 , 2 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 23); - _mm256_storeu_si256(out + 27, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 25) ,_mm256_slli_epi32( w1 , 7 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 24); - _mm256_storeu_si256(out + 28, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 20) ,_mm256_slli_epi32( w0 , 12 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 25); - _mm256_storeu_si256(out + 29, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 15) ,_mm256_slli_epi32( w1 , 17 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 26); - _mm256_storeu_si256(out + 30, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 10) ,_mm256_slli_epi32( w0 , 22 ) ) ) ); - _mm256_storeu_si256(out + 31, _mm256_srli_epi32( w0 , 5) ); -} - - -/* we packed 256 28-bit values, touching 28 256-bit words, using 448 bytes */ -static void avxunpackblock28(const __m256i * compressed, uint32_t * pout) { - /* we are going to access 28 256-bit words */ - __m256i w0, w1; - __m256i * out = (__m256i *) pout; - const __m256i mask = _mm256_set1_epi32(268435455); - w0 = _mm256_lddqu_si256 (compressed); - _mm256_storeu_si256(out + 0, _mm256_and_si256 ( mask, w0 ) ); - w1 = _mm256_lddqu_si256 (compressed + 1); - _mm256_storeu_si256(out + 1, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 28) ,_mm256_slli_epi32( w1 , 4 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 2); - _mm256_storeu_si256(out + 2, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 24) ,_mm256_slli_epi32( w0 , 8 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 3); - _mm256_storeu_si256(out + 3, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 20) ,_mm256_slli_epi32( w1 , 12 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 4); - _mm256_storeu_si256(out + 4, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 16) ,_mm256_slli_epi32( w0 , 16 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 5); - _mm256_storeu_si256(out + 5, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 12) ,_mm256_slli_epi32( w1 , 20 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 6); - _mm256_storeu_si256(out + 6, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 8) ,_mm256_slli_epi32( w0 , 24 ) ) ) ); - _mm256_storeu_si256(out + 7, _mm256_srli_epi32( w0 , 4) ); - w1 = _mm256_lddqu_si256 (compressed + 7); - _mm256_storeu_si256(out + 8, _mm256_and_si256 ( mask, w1 ) ); - w0 = _mm256_lddqu_si256 (compressed + 8); - _mm256_storeu_si256(out + 9, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 28) ,_mm256_slli_epi32( w0 , 4 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 9); - _mm256_storeu_si256(out + 10, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 24) ,_mm256_slli_epi32( w1 , 8 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 10); - _mm256_storeu_si256(out + 11, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 20) ,_mm256_slli_epi32( w0 , 12 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 11); - _mm256_storeu_si256(out + 12, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 16) ,_mm256_slli_epi32( w1 , 16 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 12); - _mm256_storeu_si256(out + 13, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 12) ,_mm256_slli_epi32( w0 , 20 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 13); - _mm256_storeu_si256(out + 14, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 8) ,_mm256_slli_epi32( w1 , 24 ) ) ) ); - _mm256_storeu_si256(out + 15, _mm256_srli_epi32( w1 , 4) ); - w0 = _mm256_lddqu_si256 (compressed + 14); - _mm256_storeu_si256(out + 16, _mm256_and_si256 ( mask, w0 ) ); - w1 = _mm256_lddqu_si256 (compressed + 15); - _mm256_storeu_si256(out + 17, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 28) ,_mm256_slli_epi32( w1 , 4 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 16); - _mm256_storeu_si256(out + 18, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 24) ,_mm256_slli_epi32( w0 , 8 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 17); - _mm256_storeu_si256(out + 19, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 20) ,_mm256_slli_epi32( w1 , 12 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 18); - _mm256_storeu_si256(out + 20, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 16) ,_mm256_slli_epi32( w0 , 16 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 19); - _mm256_storeu_si256(out + 21, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 12) ,_mm256_slli_epi32( w1 , 20 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 20); - _mm256_storeu_si256(out + 22, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 8) ,_mm256_slli_epi32( w0 , 24 ) ) ) ); - _mm256_storeu_si256(out + 23, _mm256_srli_epi32( w0 , 4) ); - w1 = _mm256_lddqu_si256 (compressed + 21); - _mm256_storeu_si256(out + 24, _mm256_and_si256 ( mask, w1 ) ); - w0 = _mm256_lddqu_si256 (compressed + 22); - _mm256_storeu_si256(out + 25, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 28) ,_mm256_slli_epi32( w0 , 4 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 23); - _mm256_storeu_si256(out + 26, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 24) ,_mm256_slli_epi32( w1 , 8 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 24); - _mm256_storeu_si256(out + 27, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 20) ,_mm256_slli_epi32( w0 , 12 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 25); - _mm256_storeu_si256(out + 28, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 16) ,_mm256_slli_epi32( w1 , 16 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 26); - _mm256_storeu_si256(out + 29, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 12) ,_mm256_slli_epi32( w0 , 20 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 27); - _mm256_storeu_si256(out + 30, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 8) ,_mm256_slli_epi32( w1 , 24 ) ) ) ); - _mm256_storeu_si256(out + 31, _mm256_srli_epi32( w1 , 4) ); -} - - -/* we packed 256 29-bit values, touching 29 256-bit words, using 464 bytes */ -static void avxunpackblock29(const __m256i * compressed, uint32_t * pout) { - /* we are going to access 29 256-bit words */ - __m256i w0, w1; - __m256i * out = (__m256i *) pout; - const __m256i mask = _mm256_set1_epi32(536870911); - w0 = _mm256_lddqu_si256 (compressed); - _mm256_storeu_si256(out + 0, _mm256_and_si256 ( mask, w0 ) ); - w1 = _mm256_lddqu_si256 (compressed + 1); - _mm256_storeu_si256(out + 1, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 29) ,_mm256_slli_epi32( w1 , 3 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 2); - _mm256_storeu_si256(out + 2, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 26) ,_mm256_slli_epi32( w0 , 6 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 3); - _mm256_storeu_si256(out + 3, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 23) ,_mm256_slli_epi32( w1 , 9 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 4); - _mm256_storeu_si256(out + 4, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 20) ,_mm256_slli_epi32( w0 , 12 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 5); - _mm256_storeu_si256(out + 5, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 17) ,_mm256_slli_epi32( w1 , 15 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 6); - _mm256_storeu_si256(out + 6, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 14) ,_mm256_slli_epi32( w0 , 18 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 7); - _mm256_storeu_si256(out + 7, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 11) ,_mm256_slli_epi32( w1 , 21 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 8); - _mm256_storeu_si256(out + 8, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 8) ,_mm256_slli_epi32( w0 , 24 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 9); - _mm256_storeu_si256(out + 9, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 5) ,_mm256_slli_epi32( w1 , 27 ) ) ) ); - _mm256_storeu_si256(out + 10, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 2) ) ); - w0 = _mm256_lddqu_si256 (compressed + 10); - _mm256_storeu_si256(out + 11, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 31) ,_mm256_slli_epi32( w0 , 1 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 11); - _mm256_storeu_si256(out + 12, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 28) ,_mm256_slli_epi32( w1 , 4 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 12); - _mm256_storeu_si256(out + 13, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 25) ,_mm256_slli_epi32( w0 , 7 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 13); - _mm256_storeu_si256(out + 14, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 22) ,_mm256_slli_epi32( w1 , 10 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 14); - _mm256_storeu_si256(out + 15, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 19) ,_mm256_slli_epi32( w0 , 13 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 15); - _mm256_storeu_si256(out + 16, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 16) ,_mm256_slli_epi32( w1 , 16 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 16); - _mm256_storeu_si256(out + 17, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 13) ,_mm256_slli_epi32( w0 , 19 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 17); - _mm256_storeu_si256(out + 18, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 10) ,_mm256_slli_epi32( w1 , 22 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 18); - _mm256_storeu_si256(out + 19, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 7) ,_mm256_slli_epi32( w0 , 25 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 19); - _mm256_storeu_si256(out + 20, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 4) ,_mm256_slli_epi32( w1 , 28 ) ) ) ); - _mm256_storeu_si256(out + 21, _mm256_and_si256 ( mask, _mm256_srli_epi32( w1 , 1) ) ); - w0 = _mm256_lddqu_si256 (compressed + 20); - _mm256_storeu_si256(out + 22, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 30) ,_mm256_slli_epi32( w0 , 2 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 21); - _mm256_storeu_si256(out + 23, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 27) ,_mm256_slli_epi32( w1 , 5 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 22); - _mm256_storeu_si256(out + 24, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 24) ,_mm256_slli_epi32( w0 , 8 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 23); - _mm256_storeu_si256(out + 25, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 21) ,_mm256_slli_epi32( w1 , 11 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 24); - _mm256_storeu_si256(out + 26, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 18) ,_mm256_slli_epi32( w0 , 14 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 25); - _mm256_storeu_si256(out + 27, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 15) ,_mm256_slli_epi32( w1 , 17 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 26); - _mm256_storeu_si256(out + 28, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 12) ,_mm256_slli_epi32( w0 , 20 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 27); - _mm256_storeu_si256(out + 29, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 9) ,_mm256_slli_epi32( w1 , 23 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 28); - _mm256_storeu_si256(out + 30, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 6) ,_mm256_slli_epi32( w0 , 26 ) ) ) ); - _mm256_storeu_si256(out + 31, _mm256_srli_epi32( w0 , 3) ); -} - - -/* we packed 256 30-bit values, touching 30 256-bit words, using 480 bytes */ -static void avxunpackblock30(const __m256i * compressed, uint32_t * pout) { - /* we are going to access 30 256-bit words */ - __m256i w0, w1; - __m256i * out = (__m256i *) pout; - const __m256i mask = _mm256_set1_epi32(1073741823); - w0 = _mm256_lddqu_si256 (compressed); - _mm256_storeu_si256(out + 0, _mm256_and_si256 ( mask, w0 ) ); - w1 = _mm256_lddqu_si256 (compressed + 1); - _mm256_storeu_si256(out + 1, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 30) ,_mm256_slli_epi32( w1 , 2 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 2); - _mm256_storeu_si256(out + 2, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 28) ,_mm256_slli_epi32( w0 , 4 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 3); - _mm256_storeu_si256(out + 3, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 26) ,_mm256_slli_epi32( w1 , 6 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 4); - _mm256_storeu_si256(out + 4, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 24) ,_mm256_slli_epi32( w0 , 8 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 5); - _mm256_storeu_si256(out + 5, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 22) ,_mm256_slli_epi32( w1 , 10 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 6); - _mm256_storeu_si256(out + 6, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 20) ,_mm256_slli_epi32( w0 , 12 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 7); - _mm256_storeu_si256(out + 7, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 18) ,_mm256_slli_epi32( w1 , 14 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 8); - _mm256_storeu_si256(out + 8, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 16) ,_mm256_slli_epi32( w0 , 16 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 9); - _mm256_storeu_si256(out + 9, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 14) ,_mm256_slli_epi32( w1 , 18 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 10); - _mm256_storeu_si256(out + 10, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 12) ,_mm256_slli_epi32( w0 , 20 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 11); - _mm256_storeu_si256(out + 11, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 10) ,_mm256_slli_epi32( w1 , 22 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 12); - _mm256_storeu_si256(out + 12, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 8) ,_mm256_slli_epi32( w0 , 24 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 13); - _mm256_storeu_si256(out + 13, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 6) ,_mm256_slli_epi32( w1 , 26 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 14); - _mm256_storeu_si256(out + 14, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 4) ,_mm256_slli_epi32( w0 , 28 ) ) ) ); - _mm256_storeu_si256(out + 15, _mm256_srli_epi32( w0 , 2) ); - w1 = _mm256_lddqu_si256 (compressed + 15); - _mm256_storeu_si256(out + 16, _mm256_and_si256 ( mask, w1 ) ); - w0 = _mm256_lddqu_si256 (compressed + 16); - _mm256_storeu_si256(out + 17, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 30) ,_mm256_slli_epi32( w0 , 2 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 17); - _mm256_storeu_si256(out + 18, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 28) ,_mm256_slli_epi32( w1 , 4 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 18); - _mm256_storeu_si256(out + 19, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 26) ,_mm256_slli_epi32( w0 , 6 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 19); - _mm256_storeu_si256(out + 20, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 24) ,_mm256_slli_epi32( w1 , 8 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 20); - _mm256_storeu_si256(out + 21, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 22) ,_mm256_slli_epi32( w0 , 10 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 21); - _mm256_storeu_si256(out + 22, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 20) ,_mm256_slli_epi32( w1 , 12 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 22); - _mm256_storeu_si256(out + 23, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 18) ,_mm256_slli_epi32( w0 , 14 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 23); - _mm256_storeu_si256(out + 24, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 16) ,_mm256_slli_epi32( w1 , 16 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 24); - _mm256_storeu_si256(out + 25, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 14) ,_mm256_slli_epi32( w0 , 18 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 25); - _mm256_storeu_si256(out + 26, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 12) ,_mm256_slli_epi32( w1 , 20 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 26); - _mm256_storeu_si256(out + 27, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 10) ,_mm256_slli_epi32( w0 , 22 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 27); - _mm256_storeu_si256(out + 28, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 8) ,_mm256_slli_epi32( w1 , 24 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 28); - _mm256_storeu_si256(out + 29, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 6) ,_mm256_slli_epi32( w0 , 26 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 29); - _mm256_storeu_si256(out + 30, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 4) ,_mm256_slli_epi32( w1 , 28 ) ) ) ); - _mm256_storeu_si256(out + 31, _mm256_srli_epi32( w1 , 2) ); -} - - -/* we packed 256 31-bit values, touching 31 256-bit words, using 496 bytes */ -static void avxunpackblock31(const __m256i * compressed, uint32_t * pout) { - /* we are going to access 31 256-bit words */ - __m256i w0, w1; - __m256i * out = (__m256i *) pout; - const __m256i mask = _mm256_set1_epi32(2147483647); - w0 = _mm256_lddqu_si256 (compressed); - _mm256_storeu_si256(out + 0, _mm256_and_si256 ( mask, w0 ) ); - w1 = _mm256_lddqu_si256 (compressed + 1); - _mm256_storeu_si256(out + 1, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 31) ,_mm256_slli_epi32( w1 , 1 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 2); - _mm256_storeu_si256(out + 2, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 30) ,_mm256_slli_epi32( w0 , 2 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 3); - _mm256_storeu_si256(out + 3, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 29) ,_mm256_slli_epi32( w1 , 3 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 4); - _mm256_storeu_si256(out + 4, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 28) ,_mm256_slli_epi32( w0 , 4 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 5); - _mm256_storeu_si256(out + 5, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 27) ,_mm256_slli_epi32( w1 , 5 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 6); - _mm256_storeu_si256(out + 6, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 26) ,_mm256_slli_epi32( w0 , 6 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 7); - _mm256_storeu_si256(out + 7, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 25) ,_mm256_slli_epi32( w1 , 7 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 8); - _mm256_storeu_si256(out + 8, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 24) ,_mm256_slli_epi32( w0 , 8 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 9); - _mm256_storeu_si256(out + 9, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 23) ,_mm256_slli_epi32( w1 , 9 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 10); - _mm256_storeu_si256(out + 10, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 22) ,_mm256_slli_epi32( w0 , 10 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 11); - _mm256_storeu_si256(out + 11, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 21) ,_mm256_slli_epi32( w1 , 11 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 12); - _mm256_storeu_si256(out + 12, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 20) ,_mm256_slli_epi32( w0 , 12 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 13); - _mm256_storeu_si256(out + 13, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 19) ,_mm256_slli_epi32( w1 , 13 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 14); - _mm256_storeu_si256(out + 14, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 18) ,_mm256_slli_epi32( w0 , 14 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 15); - _mm256_storeu_si256(out + 15, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 17) ,_mm256_slli_epi32( w1 , 15 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 16); - _mm256_storeu_si256(out + 16, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 16) ,_mm256_slli_epi32( w0 , 16 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 17); - _mm256_storeu_si256(out + 17, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 15) ,_mm256_slli_epi32( w1 , 17 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 18); - _mm256_storeu_si256(out + 18, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 14) ,_mm256_slli_epi32( w0 , 18 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 19); - _mm256_storeu_si256(out + 19, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 13) ,_mm256_slli_epi32( w1 , 19 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 20); - _mm256_storeu_si256(out + 20, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 12) ,_mm256_slli_epi32( w0 , 20 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 21); - _mm256_storeu_si256(out + 21, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 11) ,_mm256_slli_epi32( w1 , 21 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 22); - _mm256_storeu_si256(out + 22, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 10) ,_mm256_slli_epi32( w0 , 22 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 23); - _mm256_storeu_si256(out + 23, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 9) ,_mm256_slli_epi32( w1 , 23 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 24); - _mm256_storeu_si256(out + 24, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 8) ,_mm256_slli_epi32( w0 , 24 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 25); - _mm256_storeu_si256(out + 25, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 7) ,_mm256_slli_epi32( w1 , 25 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 26); - _mm256_storeu_si256(out + 26, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 6) ,_mm256_slli_epi32( w0 , 26 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 27); - _mm256_storeu_si256(out + 27, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 5) ,_mm256_slli_epi32( w1 , 27 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 28); - _mm256_storeu_si256(out + 28, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 4) ,_mm256_slli_epi32( w0 , 28 ) ) ) ); - w1 = _mm256_lddqu_si256 (compressed + 29); - _mm256_storeu_si256(out + 29, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w0 , 3) ,_mm256_slli_epi32( w1 , 29 ) ) ) ); - w0 = _mm256_lddqu_si256 (compressed + 30); - _mm256_storeu_si256(out + 30, - _mm256_and_si256 ( mask, _mm256_or_si256 (_mm256_srli_epi32( w1 , 2) ,_mm256_slli_epi32( w0 , 30 ) ) ) ); - _mm256_storeu_si256(out + 31, _mm256_srli_epi32( w0 , 1) ); -} - - -/* we packed 256 32-bit values, touching 32 256-bit words, using 512 bytes */ -static void avxunpackblock32(const __m256i * compressed, uint32_t * pout) { - /* we are going to access 32 256-bit words */ - __m256i w0, w1; - __m256i * out = (__m256i *) pout; - w0 = _mm256_lddqu_si256 (compressed); - _mm256_storeu_si256(out + 0, w0 ); - w1 = _mm256_lddqu_si256 (compressed + 1); - _mm256_storeu_si256(out + 1, w1 ); - w0 = _mm256_lddqu_si256 (compressed + 2); - _mm256_storeu_si256(out + 2, w0 ); - w1 = _mm256_lddqu_si256 (compressed + 3); - _mm256_storeu_si256(out + 3, w1 ); - w0 = _mm256_lddqu_si256 (compressed + 4); - _mm256_storeu_si256(out + 4, w0 ); - w1 = _mm256_lddqu_si256 (compressed + 5); - _mm256_storeu_si256(out + 5, w1 ); - w0 = _mm256_lddqu_si256 (compressed + 6); - _mm256_storeu_si256(out + 6, w0 ); - w1 = _mm256_lddqu_si256 (compressed + 7); - _mm256_storeu_si256(out + 7, w1 ); - w0 = _mm256_lddqu_si256 (compressed + 8); - _mm256_storeu_si256(out + 8, w0 ); - w1 = _mm256_lddqu_si256 (compressed + 9); - _mm256_storeu_si256(out + 9, w1 ); - w0 = _mm256_lddqu_si256 (compressed + 10); - _mm256_storeu_si256(out + 10, w0 ); - w1 = _mm256_lddqu_si256 (compressed + 11); - _mm256_storeu_si256(out + 11, w1 ); - w0 = _mm256_lddqu_si256 (compressed + 12); - _mm256_storeu_si256(out + 12, w0 ); - w1 = _mm256_lddqu_si256 (compressed + 13); - _mm256_storeu_si256(out + 13, w1 ); - w0 = _mm256_lddqu_si256 (compressed + 14); - _mm256_storeu_si256(out + 14, w0 ); - w1 = _mm256_lddqu_si256 (compressed + 15); - _mm256_storeu_si256(out + 15, w1 ); - w0 = _mm256_lddqu_si256 (compressed + 16); - _mm256_storeu_si256(out + 16, w0 ); - w1 = _mm256_lddqu_si256 (compressed + 17); - _mm256_storeu_si256(out + 17, w1 ); - w0 = _mm256_lddqu_si256 (compressed + 18); - _mm256_storeu_si256(out + 18, w0 ); - w1 = _mm256_lddqu_si256 (compressed + 19); - _mm256_storeu_si256(out + 19, w1 ); - w0 = _mm256_lddqu_si256 (compressed + 20); - _mm256_storeu_si256(out + 20, w0 ); - w1 = _mm256_lddqu_si256 (compressed + 21); - _mm256_storeu_si256(out + 21, w1 ); - w0 = _mm256_lddqu_si256 (compressed + 22); - _mm256_storeu_si256(out + 22, w0 ); - w1 = _mm256_lddqu_si256 (compressed + 23); - _mm256_storeu_si256(out + 23, w1 ); - w0 = _mm256_lddqu_si256 (compressed + 24); - _mm256_storeu_si256(out + 24, w0 ); - w1 = _mm256_lddqu_si256 (compressed + 25); - _mm256_storeu_si256(out + 25, w1 ); - w0 = _mm256_lddqu_si256 (compressed + 26); - _mm256_storeu_si256(out + 26, w0 ); - w1 = _mm256_lddqu_si256 (compressed + 27); - _mm256_storeu_si256(out + 27, w1 ); - w0 = _mm256_lddqu_si256 (compressed + 28); - _mm256_storeu_si256(out + 28, w0 ); - w1 = _mm256_lddqu_si256 (compressed + 29); - _mm256_storeu_si256(out + 29, w1 ); - w0 = _mm256_lddqu_si256 (compressed + 30); - _mm256_storeu_si256(out + 30, w0 ); - w1 = _mm256_lddqu_si256 (compressed + 31); - _mm256_storeu_si256(out + 31, w1 ); -} - -static avxpackblockfnc avxfuncPackArr[] = { -&avxpackblock0, -&avxpackblock1, -&avxpackblock2, -&avxpackblock3, -&avxpackblock4, -&avxpackblock5, -&avxpackblock6, -&avxpackblock7, -&avxpackblock8, -&avxpackblock9, -&avxpackblock10, -&avxpackblock11, -&avxpackblock12, -&avxpackblock13, -&avxpackblock14, -&avxpackblock15, -&avxpackblock16, -&avxpackblock17, -&avxpackblock18, -&avxpackblock19, -&avxpackblock20, -&avxpackblock21, -&avxpackblock22, -&avxpackblock23, -&avxpackblock24, -&avxpackblock25, -&avxpackblock26, -&avxpackblock27, -&avxpackblock28, -&avxpackblock29, -&avxpackblock30, -&avxpackblock31, -&avxpackblock32 -}; -static avxpackblockfnc avxfuncPackMaskArr[] = { -&avxpackblockmask0, -&avxpackblockmask1, -&avxpackblockmask2, -&avxpackblockmask3, -&avxpackblockmask4, -&avxpackblockmask5, -&avxpackblockmask6, -&avxpackblockmask7, -&avxpackblockmask8, -&avxpackblockmask9, -&avxpackblockmask10, -&avxpackblockmask11, -&avxpackblockmask12, -&avxpackblockmask13, -&avxpackblockmask14, -&avxpackblockmask15, -&avxpackblockmask16, -&avxpackblockmask17, -&avxpackblockmask18, -&avxpackblockmask19, -&avxpackblockmask20, -&avxpackblockmask21, -&avxpackblockmask22, -&avxpackblockmask23, -&avxpackblockmask24, -&avxpackblockmask25, -&avxpackblockmask26, -&avxpackblockmask27, -&avxpackblockmask28, -&avxpackblockmask29, -&avxpackblockmask30, -&avxpackblockmask31, -&avxpackblockmask32 -}; -static avxunpackblockfnc avxfuncUnpackArr[] = { -&avxunpackblock0, -&avxunpackblock1, -&avxunpackblock2, -&avxunpackblock3, -&avxunpackblock4, -&avxunpackblock5, -&avxunpackblock6, -&avxunpackblock7, -&avxunpackblock8, -&avxunpackblock9, -&avxunpackblock10, -&avxunpackblock11, -&avxunpackblock12, -&avxunpackblock13, -&avxunpackblock14, -&avxunpackblock15, -&avxunpackblock16, -&avxunpackblock17, -&avxunpackblock18, -&avxunpackblock19, -&avxunpackblock20, -&avxunpackblock21, -&avxunpackblock22, -&avxunpackblock23, -&avxunpackblock24, -&avxunpackblock25, -&avxunpackblock26, -&avxunpackblock27, -&avxunpackblock28, -&avxunpackblock29, -&avxunpackblock30, -&avxunpackblock31, -&avxunpackblock32 -}; -/** code generated by avxpacking.py ends here **/ - - - - - - - - - -/* reads 256 values from "in", writes "bit" 256-bit vectors to "out" */ -void avxpack(const uint32_t * in,__m256i * out, const uint32_t bit) { - avxfuncPackMaskArr[bit](in,out); -} - -/* reads 256 values from "in", writes "bit" 256-bit vectors to "out" */ -void avxpackwithoutmask(const uint32_t * in,__m256i * out, const uint32_t bit) { - avxfuncPackArr[bit](in,out); -} - -/* reads "bit" 256-bit vectors from "in", writes 256 values to "out" */ -void avxunpack(const __m256i * in,uint32_t * out, const uint32_t bit) { - avxfuncUnpackArr[bit](in,out); -} - -#endif /* __AVX2__ */ diff --git a/cpp/simdcomp/src/simdbitpacking.c b/cpp/simdcomp/src/simdbitpacking.c deleted file mode 100644 index d249091e9..000000000 --- a/cpp/simdcomp/src/simdbitpacking.c +++ /dev/null @@ -1,14183 +0,0 @@ -/** - * This code is released under a BSD License. - */ -#include "simdbitpacking.h" - - -static void SIMD_nullunpacker32(const __m128i * _in , uint32_t * out) { - (void) _in; - memset(out,0,32 * 4 * 4); -} - -static void __SIMD_fastpackwithoutmask1_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - __m128i InReg = _mm_loadu_si128(in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 1)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 3)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 5)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 7)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 9)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 11)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 13)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 15)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 17)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 19)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 21)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 23)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 25)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 27)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 29)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 31)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpackwithoutmask2_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - __m128i InReg = _mm_loadu_si128(in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpackwithoutmask3_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - __m128i InReg = _mm_loadu_si128(in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 3)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 9)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 15)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 21)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 27)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 3 - 1); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 1)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 7)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 13)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 19)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 25)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 31)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 3 - 2); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 5)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 11)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 17)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 23)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 29)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpackwithoutmask5_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - __m128i InReg = _mm_loadu_si128(in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 5)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 15)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 25)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 5 - 3); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 3)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 13)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 23)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 5 - 1); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 1)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 11)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 21)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 31)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 5 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 9)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 19)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 29)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 5 - 2); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 7)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 17)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 27)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpackwithoutmask6_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - __m128i InReg = _mm_loadu_si128(in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 6 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 6 - 2); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 6 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 6 - 2); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpackwithoutmask7_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - __m128i InReg = _mm_loadu_si128(in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 7)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 21)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 7 - 3); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 3)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 17)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 31)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 7 - 6); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 13)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 27)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 7 - 2); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 9)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 23)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 7 - 5); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 5)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 19)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 7 - 1); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 1)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 15)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 29)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 7 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 11)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 25)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpackwithoutmask9_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - __m128i InReg = _mm_loadu_si128(in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 9)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 27)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 9 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 13)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 31)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 9 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 17)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 9 - 3); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 3)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 21)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 9 - 7); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 7)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 25)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 9 - 2); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 11)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 29)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 9 - 6); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 15)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 9 - 1); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 1)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 19)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 9 - 5); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 5)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 23)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpackwithoutmask10_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - __m128i InReg = _mm_loadu_si128(in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 10 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 10 - 6); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 10 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 10 - 2); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 10 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 10 - 6); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 10 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 10 - 2); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpackwithoutmask11_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - __m128i InReg = _mm_loadu_si128(in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 11)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 11 - 1); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 1)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 23)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 11 - 2); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 13)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 11 - 3); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 3)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 25)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 11 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 15)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 11 - 5); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 5)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 27)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 11 - 6); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 17)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 11 - 7); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 7)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 29)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 11 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 19)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 11 - 9); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 9)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 31)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 11 - 10); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 21)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpackwithoutmask12_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - __m128i InReg = _mm_loadu_si128(in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 12 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 12 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 12 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 12 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 12 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 12 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 12 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 12 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpackwithoutmask13_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - __m128i InReg = _mm_loadu_si128(in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 13)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 13 - 7); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 7)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 13 - 1); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 1)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 27)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 13 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 21)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 13 - 2); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 15)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 13 - 9); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 9)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 13 - 3); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 3)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 29)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 13 - 10); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 23)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 13 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 17)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 13 - 11); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 11)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 13 - 5); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 5)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 31)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 13 - 12); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 25)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 13 - 6); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 19)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpackwithoutmask14_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - __m128i InReg = _mm_loadu_si128(in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 14 - 10); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 14 - 6); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 14 - 2); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 14 - 12); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 14 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 14 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 14 - 10); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 14 - 6); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 14 - 2); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 14 - 12); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 14 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 14 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpackwithoutmask15_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - __m128i InReg = _mm_loadu_si128(in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 15)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 13); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 13)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 11); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 11)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 9); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 9)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 7); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 7)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 5); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 5)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 3); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 3)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 1); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 1)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 31)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 14); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 29)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 12); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 27)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 10); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 25)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 23)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 6); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 21)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 19)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 2); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 17)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpackwithoutmask17_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - __m128i InReg = _mm_loadu_si128(in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 17)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 2); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 19)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 21)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 6); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 23)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 25)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 10); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 27)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 12); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 29)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 14); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 31)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 16); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 1); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 1)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 3); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 3)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 5); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 5)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 7); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 7)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 9); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 9)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 11); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 11)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 13); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 13)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 15); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 15)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpackwithoutmask18_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - __m128i InReg = _mm_loadu_si128(in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 12); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 16); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 2); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 6); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 10); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 14); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 12); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 16); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 2); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 6); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 10); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 14); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpackwithoutmask19_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - __m128i InReg = _mm_loadu_si128(in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 19)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 6); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 25)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 12); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 31)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 18); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 5); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 5)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 11); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 11)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 17); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 17)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 23)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 10); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 29)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 16); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 3); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 3)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 9); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 9)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 15); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 15)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 2); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 21)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 27)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 14); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 1); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 1)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 7); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 7)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 13); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 13)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpackwithoutmask20_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - __m128i InReg = _mm_loadu_si128(in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 16); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 12); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 16); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 12); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 16); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 12); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 16); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 12); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpackwithoutmask21_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - __m128i InReg = _mm_loadu_si128(in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 21)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 10); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 31)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 20); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 9); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 9)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 19); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 19)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 29)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 18); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 7); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 7)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 17); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 17)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 6); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 27)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 16); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 5); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 5)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 15); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 15)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 25)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 14); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 3); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 3)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 13); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 13)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 2); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 23)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 12); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 1); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 1)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 11); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 11)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpackwithoutmask22_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - __m128i InReg = _mm_loadu_si128(in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 12); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 2); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 14); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 16); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 6); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 18); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 20); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 10); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 12); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 2); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 14); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 16); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 6); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 18); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 20); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 10); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpackwithoutmask23_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - __m128i InReg = _mm_loadu_si128(in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 23)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 14); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 5); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 5)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 19); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 19)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 10); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 1); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 1)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 15); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 15)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 6); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 29)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 20); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 11); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 11)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 2); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 25)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 16); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 7); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 7)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 21); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 21)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 12); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 3); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 3)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 17); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 17)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 31)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 22); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 13); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 13)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 27)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 18); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 9); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 9)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpackwithoutmask24_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - __m128i InReg = _mm_loadu_si128(in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 16); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 16); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 16); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 16); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 16); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 16); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 16); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 16); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpackwithoutmask25_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - __m128i InReg = _mm_loadu_si128(in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 25)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 18); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 11); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 11)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 29)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 22); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 15); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 15)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 1); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 1)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 19); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 19)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 12); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 5); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 5)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 23); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 23)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 16); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 9); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 9)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 2); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 27)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 20); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 13); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 13)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 6); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 31)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 24); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 17); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 17)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 10); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 3); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 3)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 21); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 21)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 14); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 7); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 7)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpackwithoutmask26_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - __m128i InReg = _mm_loadu_si128(in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 20); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 14); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 2); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 22); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 16); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 10); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 24); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 18); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 12); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 6); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 20); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 14); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 2); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 22); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 16); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 10); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 24); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 18); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 12); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 6); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpackwithoutmask27_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - __m128i InReg = _mm_loadu_si128(in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 27)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 22); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 17); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 17)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 12); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 7); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 7)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 2); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 29)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 24); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 19); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 19)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 14); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 9); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 9)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 31)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 26); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 21); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 21)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 16); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 11); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 11)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 6); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 1); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 1)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 23); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 23)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 18); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 13); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 13)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 3); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 3)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 25); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 25)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 20); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 15); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 15)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 10); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 5); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 5)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpackwithoutmask28_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - __m128i InReg = _mm_loadu_si128(in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 24); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 20); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 16); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 12); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 24); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 20); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 16); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 12); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 24); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 20); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 16); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 12); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 24); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 20); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 16); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 12); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpackwithoutmask29_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - __m128i InReg = _mm_loadu_si128(in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 29)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 26); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 23); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 23)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 20); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 17); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 17)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 14); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 11); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 11)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 5); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 5)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 2); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 31)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 28); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 25); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 25)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 22); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 19); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 19)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 16); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 13); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 13)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 10); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 7); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 7)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 1); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 1)); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 27); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 27)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 24); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 21); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 21)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 18); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 15); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 15)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 12); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 9); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 9)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 6); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 3); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 3)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpackwithoutmask30_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - __m128i InReg = _mm_loadu_si128(in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 28); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 26); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 24); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 22); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 20); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 18); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 16); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 14); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 12); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 10); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 6); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 2); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 28); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 26); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 24); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 22); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 20); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 18); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 16); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 14); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 12); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 10); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 6); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 2); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpackwithoutmask31_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - __m128i InReg = _mm_loadu_si128(in); - - OutReg = InReg; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 31)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 30); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 29); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 29)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 28); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 27); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 27)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 26); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 25); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 25)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 24); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 23); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 23)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 22); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 21); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 21)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 20); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 19); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 19)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 18); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 17); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 17)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 16); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 15); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 15)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 14); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 13); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 13)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 12); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 11); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 11)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 10); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 9); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 9)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 8); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 7); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 7)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 6); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 5); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 5)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 4); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 3); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 3)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 2); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 1); - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 1)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpackwithoutmask32_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - __m128i InReg = _mm_loadu_si128(in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpackwithoutmask4_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - __m128i InReg; - uint32_t outer; - - for(outer=0; outer< 4 ;++outer) { - InReg = _mm_loadu_si128(in); - OutReg = InReg; - - InReg = _mm_loadu_si128(in+1); - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - - InReg = _mm_loadu_si128(in+2); - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - - InReg = _mm_loadu_si128(in+3); - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - - InReg = _mm_loadu_si128(in+4); - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - - InReg = _mm_loadu_si128(in+5); - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - - InReg = _mm_loadu_si128(in+6); - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - - InReg = _mm_loadu_si128(in+7); - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - - in+=8; - } - -} - - - -static void __SIMD_fastpackwithoutmask8_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - __m128i InReg; - uint32_t outer; - - for(outer=0; outer< 8 ;++outer) { - InReg = _mm_loadu_si128(in); - OutReg = InReg; - - InReg = _mm_loadu_si128(in+1); - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - - InReg = _mm_loadu_si128(in+2); - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - - InReg = _mm_loadu_si128(in+3); - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - - in+=4; - } - -} - - - -static void __SIMD_fastpackwithoutmask16_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - __m128i InReg; - uint32_t outer; - - for(outer=0; outer< 16 ;++outer) { - InReg = _mm_loadu_si128(in); - OutReg = InReg; - - InReg = _mm_loadu_si128(in+1); - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - - in+=2; - } - -} - - - -static void __SIMD_fastpack1_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - const __m128i mask = _mm_set1_epi32((1U<<1)-1); - - __m128i InReg = _mm_and_si128(_mm_loadu_si128(in), mask); - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 1)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 2)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 3)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 5)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 6)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 7)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 9)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 10)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 11)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 13)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 14)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 15)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 17)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 18)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 19)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 21)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 22)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 23)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 25)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 26)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 27)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 29)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 30)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 31)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpack2_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - const __m128i mask = _mm_set1_epi32((1U<<2)-1); - - __m128i InReg = _mm_and_si128(_mm_loadu_si128(in), mask); - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 2)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 6)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 10)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 14)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 18)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 22)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 26)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 2)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 6)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 10)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 14)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 18)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 22)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 26)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpack3_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - const __m128i mask = _mm_set1_epi32((1U<<3)-1); - - __m128i InReg = _mm_and_si128(_mm_loadu_si128(in), mask); - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 3)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 6)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 9)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 15)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 18)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 21)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 27)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 3 - 1); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 1)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 7)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 10)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 13)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 19)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 22)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 25)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 31)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 3 - 2); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 2)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 5)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 11)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 14)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 17)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 23)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 26)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 29)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpack5_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - const __m128i mask = _mm_set1_epi32((1U<<5)-1); - - __m128i InReg = _mm_and_si128(_mm_loadu_si128(in), mask); - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 5)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 10)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 15)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 25)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 5 - 3); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 3)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 13)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 18)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 23)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 5 - 1); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 1)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 6)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 11)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 21)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 26)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 31)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 5 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 9)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 14)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 19)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 29)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 5 - 2); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 2)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 7)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 17)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 22)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 27)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpack6_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - const __m128i mask = _mm_set1_epi32((1U<<6)-1); - - __m128i InReg = _mm_and_si128(_mm_loadu_si128(in), mask); - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 6)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 18)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 6 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 10)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 22)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 6 - 2); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 2)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 14)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 6)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 18)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 6 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 10)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 22)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 6 - 2); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 2)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 14)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpack7_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - const __m128i mask = _mm_set1_epi32((1U<<7)-1); - - __m128i InReg = _mm_and_si128(_mm_loadu_si128(in), mask); - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 7)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 14)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 21)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 7 - 3); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 3)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 10)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 17)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 31)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 7 - 6); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 6)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 13)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 27)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 7 - 2); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 2)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 9)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 23)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 7 - 5); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 5)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 19)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 7 - 1); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 1)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 15)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 22)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 29)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 7 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 11)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 18)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 25)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpack9_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - const __m128i mask = _mm_set1_epi32((1U<<9)-1); - - __m128i InReg = _mm_and_si128(_mm_loadu_si128(in), mask); - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 9)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 18)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 27)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 9 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 13)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 22)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 31)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 9 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 17)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 9 - 3); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 3)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 21)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 9 - 7); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 7)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 25)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 9 - 2); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 2)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 11)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 29)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 9 - 6); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 6)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 15)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 9 - 1); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 1)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 10)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 19)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 9 - 5); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 5)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 14)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 23)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpack10_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - const __m128i mask = _mm_set1_epi32((1U<<10)-1); - - __m128i InReg = _mm_and_si128(_mm_loadu_si128(in), mask); - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 10)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 10 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 18)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 10 - 6); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 6)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 10 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 14)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 10 - 2); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 2)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 10)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 10 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 18)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 10 - 6); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 6)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 10 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 14)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 10 - 2); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 2)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpack11_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - const __m128i mask = _mm_set1_epi32((1U<<11)-1); - - __m128i InReg = _mm_and_si128(_mm_loadu_si128(in), mask); - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 11)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 11 - 1); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 1)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 23)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 11 - 2); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 2)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 13)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 11 - 3); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 3)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 14)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 25)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 11 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 15)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 11 - 5); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 5)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 27)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 11 - 6); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 6)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 17)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 11 - 7); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 7)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 18)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 29)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 11 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 19)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 11 - 9); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 9)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 31)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 11 - 10); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 10)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 21)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpack12_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - const __m128i mask = _mm_set1_epi32((1U<<12)-1); - - __m128i InReg = _mm_and_si128(_mm_loadu_si128(in), mask); - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 12 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 12 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 12 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 12 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 12 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 12 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 12 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 12 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpack13_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - const __m128i mask = _mm_set1_epi32((1U<<13)-1); - - __m128i InReg = _mm_and_si128(_mm_loadu_si128(in), mask); - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 13)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 13 - 7); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 7)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 13 - 1); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 1)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 14)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 27)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 13 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 21)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 13 - 2); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 2)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 15)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 13 - 9); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 9)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 13 - 3); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 3)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 29)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 13 - 10); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 10)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 23)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 13 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 17)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 13 - 11); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 11)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 13 - 5); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 5)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 18)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 31)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 13 - 12); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 25)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 13 - 6); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 6)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 19)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpack14_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - const __m128i mask = _mm_set1_epi32((1U<<14)-1); - - __m128i InReg = _mm_and_si128(_mm_loadu_si128(in), mask); - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 14)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 14 - 10); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 10)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 14 - 6); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 6)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 14 - 2); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 2)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 14 - 12); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 14 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 14 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 14)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 14 - 10); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 10)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 14 - 6); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 6)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 14 - 2); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 2)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 14 - 12); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 14 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 14 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpack15_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - const __m128i mask = _mm_set1_epi32((1U<<15)-1); - - __m128i InReg = _mm_and_si128(_mm_loadu_si128(in), mask); - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 15)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 13); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 13)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 11); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 11)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 9); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 9)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 7); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 7)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 5); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 5)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 3); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 3)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 1); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 1)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 31)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 14); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 14)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 29)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 12); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 27)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 10); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 10)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 25)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 23)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 6); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 6)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 21)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 19)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 2); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 2)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 17)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpack17_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - const __m128i mask = _mm_set1_epi32((1U<<17)-1); - - __m128i InReg = _mm_and_si128(_mm_loadu_si128(in), mask); - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 17)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 2); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 2)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 19)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 21)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 6); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 6)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 23)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 25)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 10); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 10)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 27)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 12); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 29)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 14); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 14)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 31)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 16); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 1); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 1)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 3); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 3)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 5); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 5)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 7); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 7)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 9); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 9)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 11); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 11)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 13); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 13)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 15); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 15)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpack18_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - const __m128i mask = _mm_set1_epi32((1U<<18)-1); - - __m128i InReg = _mm_and_si128(_mm_loadu_si128(in), mask); - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 12); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 16); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 2); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 2)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 6); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 6)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 10); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 10)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 14); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 14)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 12); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 16); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 2); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 2)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 6); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 6)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 10); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 10)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 14); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 14)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpack19_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - const __m128i mask = _mm_set1_epi32((1U<<19)-1); - - __m128i InReg = _mm_and_si128(_mm_loadu_si128(in), mask); - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 19)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 6); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 6)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 25)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 12); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 31)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 18); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 5); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 5)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 11); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 11)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 17); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 17)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 23)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 10); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 10)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 29)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 16); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 3); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 3)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 9); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 9)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 15); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 15)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 2); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 2)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 21)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 27)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 14); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 14)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 1); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 1)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 7); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 7)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 13); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 13)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpack20_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - const __m128i mask = _mm_set1_epi32((1U<<20)-1); - - __m128i InReg = _mm_and_si128(_mm_loadu_si128(in), mask); - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 16); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 12); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 16); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 12); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 16); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 12); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 16); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 12); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpack21_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - const __m128i mask = _mm_set1_epi32((1U<<21)-1); - - __m128i InReg = _mm_and_si128(_mm_loadu_si128(in), mask); - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 21)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 10); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 10)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 31)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 20); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 9); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 9)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 19); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 19)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 29)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 18); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 7); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 7)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 17); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 17)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 6); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 6)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 27)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 16); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 5); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 5)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 15); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 15)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 25)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 14); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 14)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 3); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 3)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 13); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 13)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 2); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 2)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 23)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 12); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 1); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 1)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 11); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 11)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpack22_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - const __m128i mask = _mm_set1_epi32((1U<<22)-1); - - __m128i InReg = _mm_and_si128(_mm_loadu_si128(in), mask); - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 12); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 2); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 2)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 14); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 14)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 16); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 6); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 6)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 18); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 20); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 10); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 10)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 12); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 2); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 2)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 14); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 14)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 16); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 6); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 6)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 18); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 20); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 10); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 10)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpack23_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - const __m128i mask = _mm_set1_epi32((1U<<23)-1); - - __m128i InReg = _mm_and_si128(_mm_loadu_si128(in), mask); - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 23)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 14); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 14)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 5); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 5)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 19); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 19)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 10); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 10)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 1); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 1)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 15); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 15)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 6); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 6)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 29)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 20); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 11); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 11)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 2); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 2)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 25)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 16); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 7); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 7)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 21); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 21)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 12); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 3); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 3)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 17); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 17)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 31)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 22); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 13); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 13)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 27)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 18); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 9); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 9)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpack24_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - const __m128i mask = _mm_set1_epi32((1U<<24)-1); - - __m128i InReg = _mm_and_si128(_mm_loadu_si128(in), mask); - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 16); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 16); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 16); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 16); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 16); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 16); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 16); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 16); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpack25_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - const __m128i mask = _mm_set1_epi32((1U<<25)-1); - - __m128i InReg = _mm_and_si128(_mm_loadu_si128(in), mask); - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 25)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 18); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 11); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 11)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 29)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 22); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 15); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 15)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 1); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 1)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 19); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 19)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 12); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 5); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 5)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 23); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 23)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 16); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 9); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 9)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 2); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 2)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 27)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 20); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 13); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 13)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 6); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 6)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 31)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 24); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 17); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 17)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 10); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 10)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 3); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 3)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 21); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 21)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 14); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 14)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 7); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 7)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpack26_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - const __m128i mask = _mm_set1_epi32((1U<<26)-1); - - __m128i InReg = _mm_and_si128(_mm_loadu_si128(in), mask); - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 20); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 14); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 14)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 2); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 2)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 22); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 16); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 10); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 10)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 24); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 18); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 12); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 6); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 6)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 20); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 14); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 14)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 2); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 2)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 22); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 16); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 10); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 10)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 24); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 18); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 12); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 6); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 6)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpack27_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - const __m128i mask = _mm_set1_epi32((1U<<27)-1); - - __m128i InReg = _mm_and_si128(_mm_loadu_si128(in), mask); - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 27)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 22); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 17); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 17)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 12); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 7); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 7)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 2); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 2)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 29)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 24); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 19); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 19)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 14); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 14)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 9); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 9)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 31)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 26); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 21); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 21)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 16); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 11); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 11)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 6); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 6)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 1); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 1)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 23); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 23)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 18); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 13); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 13)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 3); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 3)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 25); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 25)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 20); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 15); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 15)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 10); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 10)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 5); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 5)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpack28_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - const __m128i mask = _mm_set1_epi32((1U<<28)-1); - - __m128i InReg = _mm_and_si128(_mm_loadu_si128(in), mask); - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 24); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 20); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 16); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 12); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 24); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 20); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 16); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 12); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 24); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 20); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 16); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 12); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 24); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 20); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 16); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 12); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpack29_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - const __m128i mask = _mm_set1_epi32((1U<<29)-1); - - __m128i InReg = _mm_and_si128(_mm_loadu_si128(in), mask); - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 29)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 26); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 23); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 23)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 20); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 17); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 17)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 14); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 14)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 11); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 11)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 5); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 5)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 2); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 2)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 31)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 28); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 25); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 25)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 22); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 19); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 19)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 16); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 13); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 13)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 10); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 10)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 7); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 7)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 1); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 1)); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 27); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 27)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 24); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 21); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 21)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 18); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 15); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 15)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 12); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 9); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 9)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 6); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 6)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 3); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 3)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpack30_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - const __m128i mask = _mm_set1_epi32((1U<<30)-1); - - __m128i InReg = _mm_and_si128(_mm_loadu_si128(in), mask); - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 28); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 26); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 24); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 22); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 20); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 18); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 16); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 14); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 14)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 12); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 10); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 10)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 6); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 6)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 2); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 2)); - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 28); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 26); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 24); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 22); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 20); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 18); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 16); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 14); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 14)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 12); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 10); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 10)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 6); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 6)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 2); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 2)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpack31_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - const __m128i mask = _mm_set1_epi32((1U<<31)-1); - - __m128i InReg = _mm_and_si128(_mm_loadu_si128(in), mask); - OutReg = InReg; - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 31)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 30); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 30)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 29); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 29)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 28); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 27); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 27)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 26); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 26)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 25); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 25)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 24); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 23); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 23)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 22); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 22)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 21); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 21)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 20); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 20)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 19); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 19)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 18); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 18)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 17); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 17)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 16); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 15); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 15)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 14); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 14)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 13); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 13)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 12); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 12)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 11); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 11)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 10); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 10)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 9); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 9)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 8); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 8)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 7); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 7)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 6); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 6)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 5); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 5)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 4); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 4)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 3); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 3)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 2); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 2)); - _mm_storeu_si128(out, OutReg); - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 1); - InReg = _mm_and_si128(_mm_loadu_si128(++in), mask); - - OutReg = _mm_or_si128(OutReg,_mm_slli_epi32(InReg, 1)); - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpack32_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - __m128i InReg = _mm_loadu_si128(in); - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - ++out; - InReg = _mm_loadu_si128(++in); - - OutReg = InReg; - _mm_storeu_si128(out, OutReg); - - -} - - - -static void __SIMD_fastpack4_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg, InReg; - const __m128i mask = _mm_set1_epi32((1U<<4)-1); - uint32_t outer; - - - for(outer=0; outer< 4 ;++outer) { - InReg = _mm_and_si128(_mm_loadu_si128(in), mask); - OutReg = InReg; - - InReg = _mm_and_si128(_mm_loadu_si128(in+1), mask); - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - - InReg = _mm_and_si128(_mm_loadu_si128(in+2), mask); - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - - InReg = _mm_and_si128(_mm_loadu_si128(in+3), mask); - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - - InReg = _mm_and_si128(_mm_loadu_si128(in+4), mask); - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - - InReg = _mm_and_si128(_mm_loadu_si128(in+5), mask); - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - - InReg = _mm_and_si128(_mm_loadu_si128(in+6), mask); - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - - InReg = _mm_and_si128(_mm_loadu_si128(in+7), mask); - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_storeu_si128(out, OutReg); - ++out; - - in+=8; - } - -} - - - -static void __SIMD_fastpack8_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg, InReg; - const __m128i mask = _mm_set1_epi32((1U<<8)-1); - uint32_t outer; - - - for(outer=0; outer< 8 ;++outer) { - InReg = _mm_and_si128(_mm_loadu_si128(in), mask); - OutReg = InReg; - - InReg = _mm_and_si128(_mm_loadu_si128(in+1), mask); - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - - InReg = _mm_and_si128(_mm_loadu_si128(in+2), mask); - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - - InReg = _mm_and_si128(_mm_loadu_si128(in+3), mask); - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_storeu_si128(out, OutReg); - ++out; - - in+=4; - } - -} - - - -static void __SIMD_fastpack16_32(const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg, InReg; - const __m128i mask = _mm_set1_epi32((1U<<16)-1); - uint32_t outer; - - - for(outer=0; outer< 16 ;++outer) { - InReg = _mm_and_si128(_mm_loadu_si128(in), mask); - OutReg = InReg; - - InReg = _mm_and_si128(_mm_loadu_si128(in+1), mask); - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_storeu_si128(out, OutReg); - ++out; - - in+=2; - } - -} - - - - -static void __SIMD_fastunpack1_32(const __m128i* in, uint32_t * _out) { - __m128i* out = (__m128i*)(_out); - __m128i InReg1 = _mm_loadu_si128(in); - __m128i InReg2 = InReg1; - __m128i OutReg1, OutReg2, OutReg3, OutReg4; - const __m128i mask = _mm_set1_epi32(1); - - uint32_t i, shift = 0; - - for (i = 0; i < 8; ++i) { - OutReg1 = _mm_and_si128( _mm_srli_epi32(InReg1,shift++) , mask); - OutReg2 = _mm_and_si128( _mm_srli_epi32(InReg2,shift++) , mask); - OutReg3 = _mm_and_si128( _mm_srli_epi32(InReg1,shift++) , mask); - OutReg4 = _mm_and_si128( _mm_srli_epi32(InReg2,shift++) , mask); - _mm_storeu_si128(out++, OutReg1); - _mm_storeu_si128(out++, OutReg2); - _mm_storeu_si128(out++, OutReg3); - _mm_storeu_si128(out++, OutReg4); - } -} - - - - -static void __SIMD_fastunpack2_32(const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_loadu_si128(in); - __m128i OutReg; - const __m128i mask = _mm_set1_epi32((1U<<2)-1); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,2) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,6) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,10) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,12) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,14) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,16) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,18) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,20) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,22) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,24) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,26) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,28) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,30) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,2) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,6) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,10) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,12) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,14) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,16) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,18) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,20) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,22) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,24) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,26) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,28) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,30) ; - _mm_storeu_si128(out++, OutReg); - - -} - - - - -static void __SIMD_fastunpack3_32(const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_loadu_si128(in); - __m128i OutReg; - const __m128i mask = _mm_set1_epi32((1U<<3)-1); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,3) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,6) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,9) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,12) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,15) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,18) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,21) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,24) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,27) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,30) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 3-1), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,1) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,7) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,10) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,13) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,16) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,19) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,22) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,25) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,28) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,31) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 3-2), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,2) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,5) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,11) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,14) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,17) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,20) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,23) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,26) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,29) ; - _mm_storeu_si128(out++, OutReg); - - -} - - - - -static void __SIMD_fastunpack4_32(const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_loadu_si128(in); - __m128i OutReg; - const __m128i mask = _mm_set1_epi32((1U<<4)-1); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,12) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,16) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,20) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,24) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,12) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,16) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,20) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,24) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,12) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,16) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,20) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,24) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,12) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,16) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,20) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,24) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - _mm_storeu_si128(out++, OutReg); - - -} - - - - -static void __SIMD_fastunpack5_32(const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_loadu_si128(in); - __m128i OutReg; - const __m128i mask = _mm_set1_epi32((1U<<5)-1); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,5) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,10) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,15) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,20) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,25) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,30) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 5-3), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,3) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,13) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,18) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,23) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 5-1), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,1) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,6) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,11) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,16) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,21) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,26) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,31) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 5-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,9) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,14) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,19) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,24) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,29) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 5-2), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,2) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,7) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,12) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,17) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,22) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,27) ; - _mm_storeu_si128(out++, OutReg); - - -} - - - - -static void __SIMD_fastunpack6_32(const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_loadu_si128(in); - __m128i OutReg; - const __m128i mask = _mm_set1_epi32((1U<<6)-1); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,6) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,12) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,18) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,24) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,30) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 6-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,10) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,16) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,22) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 6-2), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,2) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,14) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,20) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,26) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,6) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,12) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,18) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,24) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,30) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 6-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,10) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,16) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,22) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 6-2), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,2) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,14) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,20) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,26) ; - _mm_storeu_si128(out++, OutReg); - - -} - - - - -static void __SIMD_fastunpack7_32(const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_loadu_si128(in); - __m128i OutReg; - const __m128i mask = _mm_set1_epi32((1U<<7)-1); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,7) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,14) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,21) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 7-3), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,3) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,10) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,17) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,24) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,31) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 7-6), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,6) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,13) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,20) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,27) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 7-2), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,2) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,9) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,16) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,23) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,30) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 7-5), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,5) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,12) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,19) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,26) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 7-1), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,1) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,15) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,22) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,29) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 7-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,11) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,18) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,25) ; - _mm_storeu_si128(out++, OutReg); - - -} - - - - -static void __SIMD_fastunpack8_32(const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_loadu_si128(in); - __m128i OutReg; - const __m128i mask = _mm_set1_epi32((1U<<8)-1); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,16) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,16) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,16) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,16) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,16) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,16) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,16) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,16) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - _mm_storeu_si128(out++, OutReg); - - -} - - - - -static void __SIMD_fastunpack9_32(const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_loadu_si128(in); - __m128i OutReg; - const __m128i mask = _mm_set1_epi32((1U<<9)-1); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,9) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,18) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,27) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 9-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,13) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,22) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,31) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 9-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,17) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,26) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 9-3), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,3) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,12) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,21) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,30) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 9-7), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,7) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,16) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,25) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 9-2), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,2) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,11) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,20) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,29) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 9-6), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,6) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,15) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 9-1), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,1) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,10) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,19) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 9-5), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,5) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,14) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,23) ; - _mm_storeu_si128(out++, OutReg); - - -} - - - - -static void __SIMD_fastunpack10_32(const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_loadu_si128(in); - __m128i OutReg; - const __m128i mask = _mm_set1_epi32((1U<<10)-1); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,10) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,20) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,30) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 10-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,18) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 10-6), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,6) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,16) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,26) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 10-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,14) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 10-2), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,2) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,12) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,22) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,10) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,20) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,30) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 10-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,18) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 10-6), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,6) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,16) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,26) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 10-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,14) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 10-2), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,2) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,12) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,22) ; - _mm_storeu_si128(out++, OutReg); - - -} - - - - -static void __SIMD_fastunpack11_32(const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_loadu_si128(in); - __m128i OutReg; - const __m128i mask = _mm_set1_epi32((1U<<11)-1); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,11) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,22) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 11-1), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,1) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,12) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,23) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 11-2), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,2) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,13) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 11-3), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,3) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,14) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,25) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 11-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,15) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,26) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 11-5), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,5) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,16) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,27) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 11-6), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,6) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,17) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 11-7), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,7) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,18) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,29) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 11-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,19) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,30) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 11-9), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,9) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,20) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,31) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 11-10), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,10) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,21) ; - _mm_storeu_si128(out++, OutReg); - - -} - - - - -static void __SIMD_fastunpack12_32(const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_loadu_si128(in); - __m128i OutReg; - const __m128i mask = _mm_set1_epi32((1U<<12)-1); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,12) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 12-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,16) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 12-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,20) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,12) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 12-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,16) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 12-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,20) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,12) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 12-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,16) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 12-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,20) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,12) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 12-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,16) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 12-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,20) ; - _mm_storeu_si128(out++, OutReg); - - -} - - - - -static void __SIMD_fastunpack13_32(const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_loadu_si128(in); - __m128i OutReg; - const __m128i mask = _mm_set1_epi32((1U<<13)-1); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,13) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,26) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 13-7), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,7) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,20) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 13-1), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,1) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,14) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,27) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 13-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,21) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 13-2), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,2) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,15) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 13-9), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,9) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,22) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 13-3), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,3) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,16) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,29) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 13-10), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,10) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,23) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 13-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,17) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,30) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 13-11), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,11) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 13-5), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,5) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,18) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,31) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 13-12), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,12) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,25) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 13-6), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,6) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,19) ; - _mm_storeu_si128(out++, OutReg); - - -} - - - - -static void __SIMD_fastunpack14_32(const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_loadu_si128(in); - __m128i OutReg; - const __m128i mask = _mm_set1_epi32((1U<<14)-1); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,14) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 14-10), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,10) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 14-6), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,6) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,20) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 14-2), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,2) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,16) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,30) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 14-12), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,12) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,26) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 14-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,22) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 14-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,18) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,14) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 14-10), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,10) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 14-6), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,6) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,20) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 14-2), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,2) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,16) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,30) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 14-12), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,12) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,26) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 14-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,22) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 14-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,18) ; - _mm_storeu_si128(out++, OutReg); - - -} - - - - -static void __SIMD_fastunpack15_32(const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_loadu_si128(in); - __m128i OutReg; - const __m128i mask = _mm_set1_epi32((1U<<15)-1); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,15) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,30) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 15-13), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,13) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 15-11), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,11) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,26) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 15-9), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,9) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 15-7), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,7) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,22) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 15-5), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,5) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,20) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 15-3), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,3) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,18) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 15-1), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,1) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,16) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,31) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 15-14), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,14) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,29) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 15-12), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,12) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,27) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 15-10), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,10) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,25) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 15-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,23) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 15-6), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,6) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,21) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 15-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,19) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 15-2), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,2) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,17) ; - _mm_storeu_si128(out++, OutReg); - - -} - - - - -static void __SIMD_fastunpack16_32(const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_loadu_si128(in); - __m128i OutReg; - const __m128i mask = _mm_set1_epi32((1U<<16)-1); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - _mm_storeu_si128(out++, OutReg); - - -} - - - - -static void __SIMD_fastunpack17_32(const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_loadu_si128(in); - __m128i OutReg; - const __m128i mask = _mm_set1_epi32((1U<<17)-1); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,17) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 17-2), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,2) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,19) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 17-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,21) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 17-6), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,6) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,23) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 17-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,25) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 17-10), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,10) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,27) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 17-12), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,12) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,29) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 17-14), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,14) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,31) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 17-16), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 17-1), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,1) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,18) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 17-3), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,3) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,20) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 17-5), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,5) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,22) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 17-7), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,7) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 17-9), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,9) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,26) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 17-11), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,11) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 17-13), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,13) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,30) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 17-15), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,15) ; - _mm_storeu_si128(out++, OutReg); - - -} - - - - -static void __SIMD_fastunpack18_32(const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_loadu_si128(in); - __m128i OutReg; - const __m128i mask = _mm_set1_epi32((1U<<18)-1); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,18) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 18-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,22) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 18-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,26) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 18-12), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,12) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,30) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 18-16), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 18-2), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,2) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,20) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 18-6), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,6) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 18-10), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,10) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 18-14), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,14) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,18) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 18-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,22) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 18-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,26) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 18-12), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,12) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,30) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 18-16), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 18-2), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,2) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,20) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 18-6), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,6) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 18-10), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,10) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 18-14), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,14) ; - _mm_storeu_si128(out++, OutReg); - - -} - - - - -static void __SIMD_fastunpack19_32(const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_loadu_si128(in); - __m128i OutReg; - const __m128i mask = _mm_set1_epi32((1U<<19)-1); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,19) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 19-6), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,6) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,25) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 19-12), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,12) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,31) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 19-18), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,18) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 19-5), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,5) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 19-11), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,11) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,30) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 19-17), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,17) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 19-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,23) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 19-10), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,10) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,29) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 19-16), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 19-3), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,3) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,22) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 19-9), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,9) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 19-15), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,15) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 19-2), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,2) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,21) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 19-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,27) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 19-14), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,14) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 19-1), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,1) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,20) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 19-7), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,7) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,26) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 19-13), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,13) ; - _mm_storeu_si128(out++, OutReg); - - -} - - - - -static void __SIMD_fastunpack20_32(const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_loadu_si128(in); - __m128i OutReg; - const __m128i mask = _mm_set1_epi32((1U<<20)-1); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,20) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 20-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 20-16), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 20-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 20-12), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,12) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,20) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 20-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 20-16), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 20-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 20-12), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,12) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,20) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 20-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 20-16), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 20-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 20-12), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,12) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,20) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 20-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 20-16), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 20-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 20-12), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,12) ; - _mm_storeu_si128(out++, OutReg); - - -} - - - - -static void __SIMD_fastunpack21_32(const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_loadu_si128(in); - __m128i OutReg; - const __m128i mask = _mm_set1_epi32((1U<<21)-1); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,21) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-10), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,10) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,31) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-20), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,20) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-9), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,9) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,30) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-19), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,19) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,29) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-18), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,18) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-7), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,7) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-17), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,17) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-6), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,6) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,27) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-16), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-5), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,5) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,26) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-15), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,15) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,25) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-14), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,14) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-3), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,3) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-13), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,13) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-2), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,2) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,23) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-12), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,12) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-1), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,1) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,22) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-11), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,11) ; - _mm_storeu_si128(out++, OutReg); - - -} - - - - -static void __SIMD_fastunpack22_32(const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_loadu_si128(in); - __m128i OutReg; - const __m128i mask = _mm_set1_epi32((1U<<22)-1); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,22) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-12), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,12) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-2), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,2) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-14), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,14) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,26) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-16), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-6), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,6) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-18), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,18) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,30) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-20), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,20) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-10), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,10) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,22) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-12), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,12) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-2), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,2) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-14), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,14) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,26) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-16), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-6), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,6) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-18), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,18) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,30) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-20), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,20) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-10), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,10) ; - _mm_storeu_si128(out++, OutReg); - - -} - - - - -static void __SIMD_fastunpack23_32(const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_loadu_si128(in); - __m128i OutReg; - const __m128i mask = _mm_set1_epi32((1U<<23)-1); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,23) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-14), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,14) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-5), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,5) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-19), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,19) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-10), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,10) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-1), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,1) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-15), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,15) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-6), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,6) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,29) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-20), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,20) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-11), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,11) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-2), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,2) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,25) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-16), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-7), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,7) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,30) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-21), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,21) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-12), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,12) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-3), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,3) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,26) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-17), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,17) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,8) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,31) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-22), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,22) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-13), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,13) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,27) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-18), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,18) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-9), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,9) ; - _mm_storeu_si128(out++, OutReg); - - -} - - - - -static void __SIMD_fastunpack24_32(const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_loadu_si128(in); - __m128i OutReg; - const __m128i mask = _mm_set1_epi32((1U<<24)-1); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,8) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,8) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,8) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,8) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,8) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,8) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,8) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,8) ; - _mm_storeu_si128(out++, OutReg); - - -} - - - - -static void __SIMD_fastunpack25_32(const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_loadu_si128(in); - __m128i OutReg; - const __m128i mask = _mm_set1_epi32((1U<<25)-1); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,25) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-18), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,18) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-11), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,11) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,29) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-22), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,22) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-15), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,15) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,8) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-1), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,1) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,26) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-19), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,19) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-12), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,12) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-5), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,5) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,30) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-23), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,23) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-16), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-9), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,9) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-2), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,2) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,27) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-20), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,20) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-13), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,13) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-6), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,6) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,31) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-24), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-17), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,17) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-10), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,10) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-3), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,3) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-21), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,21) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-14), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,14) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-7), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,7) ; - _mm_storeu_si128(out++, OutReg); - - -} - - - - -static void __SIMD_fastunpack26_32(const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_loadu_si128(in); - __m128i OutReg; - const __m128i mask = _mm_set1_epi32((1U<<26)-1); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,26) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-20), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,20) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-14), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,14) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,8) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-2), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,2) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-22), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,22) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-16), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-10), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,10) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,30) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-24), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-18), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,18) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-12), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,12) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-6), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,6) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,26) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-20), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,20) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-14), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,14) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,8) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-2), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,2) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-22), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,22) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-16), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-10), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,10) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,30) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-24), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-18), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,18) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-12), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,12) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-6), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,6) ; - _mm_storeu_si128(out++, OutReg); - - -} - - - - -static void __SIMD_fastunpack27_32(const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_loadu_si128(in); - __m128i OutReg; - const __m128i mask = _mm_set1_epi32((1U<<27)-1); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,27) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-22), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,22) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-17), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,17) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-12), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,12) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-7), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,7) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-2), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,2) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,29) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-24), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-19), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,19) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-14), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,14) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-9), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,9) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,4) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,31) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-26), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,26) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-21), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,21) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-16), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-11), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,11) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-6), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,6) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-1), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,1) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-23), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,23) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-18), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,18) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-13), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,13) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,8) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-3), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,3) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,30) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-25), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,25) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-20), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,20) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-15), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,15) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-10), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,10) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-5), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,5) ; - _mm_storeu_si128(out++, OutReg); - - -} - - - - -static void __SIMD_fastunpack28_32(const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_loadu_si128(in); - __m128i OutReg; - const __m128i mask = _mm_set1_epi32((1U<<28)-1); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-24), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-20), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,20) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-16), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-12), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,12) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,8) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,4) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-24), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-20), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,20) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-16), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-12), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,12) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,8) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,4) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-24), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-20), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,20) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-16), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-12), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,12) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,8) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,4) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-24), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-20), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,20) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-16), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-12), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,12) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,8) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,4) ; - _mm_storeu_si128(out++, OutReg); - - -} - - - - -static void __SIMD_fastunpack29_32(const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_loadu_si128(in); - __m128i OutReg; - const __m128i mask = _mm_set1_epi32((1U<<29)-1); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,29) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-26), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,26) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-23), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,23) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-20), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,20) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-17), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,17) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-14), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,14) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-11), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,11) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,8) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-5), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,5) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-2), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,2) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,31) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-28), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-25), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,25) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-22), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,22) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-19), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,19) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-16), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-13), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,13) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-10), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,10) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-7), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,7) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,4) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-1), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( _mm_srli_epi32(InReg,1) , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,30) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-27), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,27) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-24), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-21), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,21) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-18), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,18) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-15), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,15) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-12), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,12) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-9), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,9) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-6), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,6) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-3), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,3) ; - _mm_storeu_si128(out++, OutReg); - - -} - - - - -static void __SIMD_fastunpack30_32(const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_loadu_si128(in); - __m128i OutReg; - const __m128i mask = _mm_set1_epi32((1U<<30)-1); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,30) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-28), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-26), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,26) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-24), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-22), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,22) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-20), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,20) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-18), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,18) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-16), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-14), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,14) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-12), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,12) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-10), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,10) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,8) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-6), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,6) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,4) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-2), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,2) ; - InReg = _mm_loadu_si128(++in); - - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,30) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-28), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-26), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,26) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-24), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-22), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,22) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-20), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,20) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-18), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,18) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-16), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-14), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,14) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-12), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,12) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-10), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,10) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,8) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-6), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,6) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,4) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-2), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,2) ; - _mm_storeu_si128(out++, OutReg); - - -} - - - - -static void __SIMD_fastunpack31_32(const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_loadu_si128(in); - __m128i OutReg; - const __m128i mask = _mm_set1_epi32((1U<<31)-1); - - OutReg = _mm_and_si128( InReg , mask); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,31) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-30), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,30) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-29), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,29) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-28), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,28) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-27), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,27) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-26), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,26) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-25), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,25) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-24), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,24) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-23), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,23) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-22), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,22) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-21), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,21) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-20), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,20) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-19), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,19) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-18), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,18) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-17), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,17) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-16), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,16) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-15), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,15) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-14), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,14) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-13), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,13) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-12), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,12) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-11), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,11) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-10), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,10) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-9), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,9) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-8), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,8) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-7), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,7) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-6), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,6) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-5), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,5) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-4), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,4) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-3), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,3) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-2), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,2) ; - InReg = _mm_loadu_si128(++in); - - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-1), mask)); - _mm_storeu_si128(out++, OutReg); - - OutReg = _mm_srli_epi32(InReg,1) ; - _mm_storeu_si128(out++, OutReg); - - -} - - -void __SIMD_fastunpack32_32(const __m128i* in, uint32_t * _out) { - __m128i* out = (__m128i*)(_out); - uint32_t outer; - - for(outer=0; outer< 32 ;++outer) { - _mm_storeu_si128(out++, _mm_loadu_si128(in++)); - } -} - - - -void simdunpack(const __m128i * in, uint32_t * out, const uint32_t bit) { - switch(bit) { - case 0: SIMD_nullunpacker32(in,out); return; - - case 1: __SIMD_fastunpack1_32(in,out); return; - - case 2: __SIMD_fastunpack2_32(in,out); return; - - case 3: __SIMD_fastunpack3_32(in,out); return; - - case 4: __SIMD_fastunpack4_32(in,out); return; - - case 5: __SIMD_fastunpack5_32(in,out); return; - - case 6: __SIMD_fastunpack6_32(in,out); return; - - case 7: __SIMD_fastunpack7_32(in,out); return; - - case 8: __SIMD_fastunpack8_32(in,out); return; - - case 9: __SIMD_fastunpack9_32(in,out); return; - - case 10: __SIMD_fastunpack10_32(in,out); return; - - case 11: __SIMD_fastunpack11_32(in,out); return; - - case 12: __SIMD_fastunpack12_32(in,out); return; - - case 13: __SIMD_fastunpack13_32(in,out); return; - - case 14: __SIMD_fastunpack14_32(in,out); return; - - case 15: __SIMD_fastunpack15_32(in,out); return; - - case 16: __SIMD_fastunpack16_32(in,out); return; - - case 17: __SIMD_fastunpack17_32(in,out); return; - - case 18: __SIMD_fastunpack18_32(in,out); return; - - case 19: __SIMD_fastunpack19_32(in,out); return; - - case 20: __SIMD_fastunpack20_32(in,out); return; - - case 21: __SIMD_fastunpack21_32(in,out); return; - - case 22: __SIMD_fastunpack22_32(in,out); return; - - case 23: __SIMD_fastunpack23_32(in,out); return; - - case 24: __SIMD_fastunpack24_32(in,out); return; - - case 25: __SIMD_fastunpack25_32(in,out); return; - - case 26: __SIMD_fastunpack26_32(in,out); return; - - case 27: __SIMD_fastunpack27_32(in,out); return; - - case 28: __SIMD_fastunpack28_32(in,out); return; - - case 29: __SIMD_fastunpack29_32(in,out); return; - - case 30: __SIMD_fastunpack30_32(in,out); return; - - case 31: __SIMD_fastunpack31_32(in,out); return; - - case 32: __SIMD_fastunpack32_32(in,out); return; - - default: break; - } -} - - - - /*assumes that integers fit in the prescribed number of bits*/ -void simdpackwithoutmask(const uint32_t * in, __m128i * out, const uint32_t bit) { - switch(bit) { - case 0: return; - - case 1: __SIMD_fastpackwithoutmask1_32(in,out); return; - - case 2: __SIMD_fastpackwithoutmask2_32(in,out); return; - - case 3: __SIMD_fastpackwithoutmask3_32(in,out); return; - - case 4: __SIMD_fastpackwithoutmask4_32(in,out); return; - - case 5: __SIMD_fastpackwithoutmask5_32(in,out); return; - - case 6: __SIMD_fastpackwithoutmask6_32(in,out); return; - - case 7: __SIMD_fastpackwithoutmask7_32(in,out); return; - - case 8: __SIMD_fastpackwithoutmask8_32(in,out); return; - - case 9: __SIMD_fastpackwithoutmask9_32(in,out); return; - - case 10: __SIMD_fastpackwithoutmask10_32(in,out); return; - - case 11: __SIMD_fastpackwithoutmask11_32(in,out); return; - - case 12: __SIMD_fastpackwithoutmask12_32(in,out); return; - - case 13: __SIMD_fastpackwithoutmask13_32(in,out); return; - - case 14: __SIMD_fastpackwithoutmask14_32(in,out); return; - - case 15: __SIMD_fastpackwithoutmask15_32(in,out); return; - - case 16: __SIMD_fastpackwithoutmask16_32(in,out); return; - - case 17: __SIMD_fastpackwithoutmask17_32(in,out); return; - - case 18: __SIMD_fastpackwithoutmask18_32(in,out); return; - - case 19: __SIMD_fastpackwithoutmask19_32(in,out); return; - - case 20: __SIMD_fastpackwithoutmask20_32(in,out); return; - - case 21: __SIMD_fastpackwithoutmask21_32(in,out); return; - - case 22: __SIMD_fastpackwithoutmask22_32(in,out); return; - - case 23: __SIMD_fastpackwithoutmask23_32(in,out); return; - - case 24: __SIMD_fastpackwithoutmask24_32(in,out); return; - - case 25: __SIMD_fastpackwithoutmask25_32(in,out); return; - - case 26: __SIMD_fastpackwithoutmask26_32(in,out); return; - - case 27: __SIMD_fastpackwithoutmask27_32(in,out); return; - - case 28: __SIMD_fastpackwithoutmask28_32(in,out); return; - - case 29: __SIMD_fastpackwithoutmask29_32(in,out); return; - - case 30: __SIMD_fastpackwithoutmask30_32(in,out); return; - - case 31: __SIMD_fastpackwithoutmask31_32(in,out); return; - - case 32: __SIMD_fastpackwithoutmask32_32(in,out); return; - - default: break; - } -} - - - - /*assumes that integers fit in the prescribed number of bits*/ -void simdpack(const uint32_t * in, __m128i * out, const uint32_t bit) { - switch(bit) { - case 0: return; - - case 1: __SIMD_fastpack1_32(in,out); return; - - case 2: __SIMD_fastpack2_32(in,out); return; - - case 3: __SIMD_fastpack3_32(in,out); return; - - case 4: __SIMD_fastpack4_32(in,out); return; - - case 5: __SIMD_fastpack5_32(in,out); return; - - case 6: __SIMD_fastpack6_32(in,out); return; - - case 7: __SIMD_fastpack7_32(in,out); return; - - case 8: __SIMD_fastpack8_32(in,out); return; - - case 9: __SIMD_fastpack9_32(in,out); return; - - case 10: __SIMD_fastpack10_32(in,out); return; - - case 11: __SIMD_fastpack11_32(in,out); return; - - case 12: __SIMD_fastpack12_32(in,out); return; - - case 13: __SIMD_fastpack13_32(in,out); return; - - case 14: __SIMD_fastpack14_32(in,out); return; - - case 15: __SIMD_fastpack15_32(in,out); return; - - case 16: __SIMD_fastpack16_32(in,out); return; - - case 17: __SIMD_fastpack17_32(in,out); return; - - case 18: __SIMD_fastpack18_32(in,out); return; - - case 19: __SIMD_fastpack19_32(in,out); return; - - case 20: __SIMD_fastpack20_32(in,out); return; - - case 21: __SIMD_fastpack21_32(in,out); return; - - case 22: __SIMD_fastpack22_32(in,out); return; - - case 23: __SIMD_fastpack23_32(in,out); return; - - case 24: __SIMD_fastpack24_32(in,out); return; - - case 25: __SIMD_fastpack25_32(in,out); return; - - case 26: __SIMD_fastpack26_32(in,out); return; - - case 27: __SIMD_fastpack27_32(in,out); return; - - case 28: __SIMD_fastpack28_32(in,out); return; - - case 29: __SIMD_fastpack29_32(in,out); return; - - case 30: __SIMD_fastpack30_32(in,out); return; - - case 31: __SIMD_fastpack31_32(in,out); return; - - case 32: __SIMD_fastpack32_32(in,out); return; - - default: break; - } -} - - - -__m128i * simdpack_shortlength( const uint32_t * in, int length, __m128i * out, const uint32_t bit) { - int k; - int inwordpointer; - __m128i P; - uint32_t firstpass; - if(bit == 0) return out;/* nothing to do */ - if(bit == 32) { - memcpy(out,in,length*sizeof(uint32_t)); - return (__m128i *)((uint32_t *) out + length); - } - inwordpointer = 0; - P = _mm_setzero_si128(); - for(k = 0; k < length / 4 ; ++k) { - __m128i value = _mm_loadu_si128(((const __m128i * ) in + k)); - P = _mm_or_si128(P,_mm_slli_epi32(value, inwordpointer)); - firstpass = sizeof(uint32_t) * 8 - inwordpointer; - if(bit> firstbits;/* we write */ - return; - } -} - -int simdpack_compressedbytes(int length, const uint32_t bit) { - if(bit == 0) return 0;/* nothing to do */ - if(bit == 32) { - return length * sizeof(uint32_t); - } - return (((length + 3 )/ 4) * bit + 31 ) / 32 * sizeof(__m128i); -} - -__m128i * simdpack_length(const uint32_t * in, size_t length, __m128i * out, const uint32_t bit) { - size_t k; - for(k = 0; k < length / SIMDBlockSize; ++k) { - simdpack(in, out, bit); - in += SIMDBlockSize; - out += bit; - } - return simdpack_shortlength(in, length % SIMDBlockSize, out, bit); -} - -const __m128i * simdunpack_length(const __m128i * in, size_t length, uint32_t * out, const uint32_t bit) { - size_t k; - for(k = 0; k < length / SIMDBlockSize; ++k) { - simdunpack(in, out, bit); - out += SIMDBlockSize; - in += bit; - } - return simdunpack_shortlength(in, length % SIMDBlockSize, out, bit); -} diff --git a/cpp/simdcomp/src/simdcomputil.c b/cpp/simdcomp/src/simdcomputil.c deleted file mode 100644 index 8cdca6d84..000000000 --- a/cpp/simdcomp/src/simdcomputil.c +++ /dev/null @@ -1,234 +0,0 @@ -/** - * This code is released under a BSD License. - */ - -#include "simdcomputil.h" -#ifdef __SSE4_1__ -#include -#endif -#include - -#define Delta(curr, prev) \ - _mm_sub_epi32(curr, \ - _mm_or_si128(_mm_slli_si128(curr, 4), _mm_srli_si128(prev, 12))) - -/* returns the integer logarithm of v (bit width) */ -uint32_t bits(const uint32_t v) { -#ifdef _MSC_VER - unsigned long answer; - if (v == 0) { - return 0; - } - _BitScanReverse(&answer, v); - return answer + 1; -#else - return v == 0 ? 0 : 32 - __builtin_clz(v); /* assume GCC-like compiler if not microsoft */ -#endif -} - - - -static uint32_t maxbitas32int(const __m128i accumulator) { - const __m128i _tmp1 = _mm_or_si128(_mm_srli_si128(accumulator, 8), accumulator); /* (A,B,C,D) xor (0,0,A,B) = (A,B,C xor A,D xor B)*/ - const __m128i _tmp2 = _mm_or_si128(_mm_srli_si128(_tmp1, 4), _tmp1); /* (A,B,C xor A,D xor B) xor (0,0,0,C xor A)*/ - uint32_t ans = _mm_cvtsi128_si32(_tmp2); - return bits(ans); -} - -SIMDCOMP_PURE uint32_t maxbits(const uint32_t * begin) { - const __m128i* pin = (const __m128i*)(begin); - __m128i accumulator = _mm_loadu_si128(pin); - uint32_t k = 1; - for(; 4*k < SIMDBlockSize; ++k) { - __m128i newvec = _mm_loadu_si128(pin+k); - accumulator = _mm_or_si128(accumulator,newvec); - } - return maxbitas32int(accumulator); -} -static uint32_t orasint(const __m128i accumulator) { - const __m128i _tmp1 = _mm_or_si128(_mm_srli_si128(accumulator, 8), accumulator); /* (A,B,C,D) xor (0,0,A,B) = (A,B,C xor A,D xor B)*/ - const __m128i _tmp2 = _mm_or_si128(_mm_srli_si128(_tmp1, 4), _tmp1); /* (A,B,C xor A,D xor B) xor (0,0,0,C xor A)*/ - return _mm_cvtsi128_si32(_tmp2); -} - -#ifdef __SSE4_1__ - -static uint32_t minasint(const __m128i accumulator) { - const __m128i _tmp1 = _mm_min_epu32(_mm_srli_si128(accumulator, 8), accumulator); /* (A,B,C,D) xor (0,0,A,B) = (A,B,C xor A,D xor B)*/ - const __m128i _tmp2 = _mm_min_epu32(_mm_srli_si128(_tmp1, 4), _tmp1); /* (A,B,C xor A,D xor B) xor (0,0,0,C xor A)*/ - return _mm_cvtsi128_si32(_tmp2); -} - -static uint32_t maxasint(const __m128i accumulator) { - const __m128i _tmp1 = _mm_max_epu32(_mm_srli_si128(accumulator, 8), accumulator); /* (A,B,C,D) xor (0,0,A,B) = (A,B,C xor A,D xor B)*/ - const __m128i _tmp2 = _mm_max_epu32(_mm_srli_si128(_tmp1, 4), _tmp1); /* (A,B,C xor A,D xor B) xor (0,0,0,C xor A)*/ - return _mm_cvtsi128_si32(_tmp2); -} - -uint32_t simdmin(const uint32_t * in) { - const __m128i* pin = (const __m128i*)(in); - __m128i accumulator = _mm_loadu_si128(pin); - uint32_t k = 1; - for(; 4*k < SIMDBlockSize; ++k) { - __m128i newvec = _mm_loadu_si128(pin+k); - accumulator = _mm_min_epu32(accumulator,newvec); - } - return minasint(accumulator); -} - -void simdmaxmin(const uint32_t * in, uint32_t * getmin, uint32_t * getmax) { - const __m128i* pin = (const __m128i*)(in); - __m128i minaccumulator = _mm_loadu_si128(pin); - __m128i maxaccumulator = minaccumulator; - uint32_t k = 1; - for(; 4*k < SIMDBlockSize; ++k) { - __m128i newvec = _mm_loadu_si128(pin+k); - minaccumulator = _mm_min_epu32(minaccumulator,newvec); - maxaccumulator = _mm_max_epu32(maxaccumulator,newvec); - } - *getmin = minasint(minaccumulator); - *getmax = maxasint(maxaccumulator); -} - - -uint32_t simdmin_length(const uint32_t * in, uint32_t length) { - uint32_t currentmin = 0xFFFFFFFF; - uint32_t lengthdividedby4 = length / 4; - uint32_t offset = lengthdividedby4 * 4; - uint32_t k; - if (lengthdividedby4 > 0) { - const __m128i* pin = (const __m128i*)(in); - __m128i accumulator = _mm_loadu_si128(pin); - k = 1; - for(; 4*k < lengthdividedby4 * 4; ++k) { - __m128i newvec = _mm_loadu_si128(pin+k); - accumulator = _mm_min_epu32(accumulator,newvec); - } - currentmin = minasint(accumulator); - } - for (k = offset; k < length; ++k) - if (in[k] < currentmin) - currentmin = in[k]; - return currentmin; -} - -void simdmaxmin_length(const uint32_t * in, uint32_t length, uint32_t * getmin, uint32_t * getmax) { - uint32_t lengthdividedby4 = length / 4; - uint32_t offset = lengthdividedby4 * 4; - uint32_t k; - *getmin = 0xFFFFFFFF; - *getmax = 0; - if (lengthdividedby4 > 0) { - const __m128i* pin = (const __m128i*)(in); - __m128i minaccumulator = _mm_loadu_si128(pin); - __m128i maxaccumulator = minaccumulator; - k = 1; - for(; 4*k < lengthdividedby4 * 4; ++k) { - __m128i newvec = _mm_loadu_si128(pin+k); - minaccumulator = _mm_min_epu32(minaccumulator,newvec); - maxaccumulator = _mm_max_epu32(maxaccumulator,newvec); - } - *getmin = minasint(minaccumulator); - *getmax = maxasint(maxaccumulator); - } - for (k = offset; k < length; ++k) { - if (in[k] < *getmin) - *getmin = in[k]; - if (in[k] > *getmax) - *getmax = in[k]; - } -} - -#endif - -SIMDCOMP_PURE uint32_t maxbits_length(const uint32_t * in,uint32_t length) { - uint32_t k; - uint32_t lengthdividedby4 = length / 4; - uint32_t offset = lengthdividedby4 * 4; - uint32_t bigxor = 0; - if(lengthdividedby4 > 0) { - const __m128i* pin = (const __m128i*)(in); - __m128i accumulator = _mm_loadu_si128(pin); - k = 1; - for(; 4*k < 4*lengthdividedby4; ++k) { - __m128i newvec = _mm_loadu_si128(pin+k); - accumulator = _mm_or_si128(accumulator,newvec); - } - bigxor = orasint(accumulator); - } - for(k = offset; k < length; ++k) - bigxor |= in[k]; - return bits(bigxor); -} - - -/* maxbit over 128 integers (SIMDBlockSize) with provided initial value */ -uint32_t simdmaxbitsd1(uint32_t initvalue, const uint32_t * in) { - __m128i initoffset = _mm_set1_epi32 (initvalue); - const __m128i* pin = (const __m128i*)(in); - __m128i newvec = _mm_loadu_si128(pin); - __m128i accumulator = Delta(newvec , initoffset); - __m128i oldvec = newvec; - uint32_t k = 1; - for(; 4*k < SIMDBlockSize; ++k) { - newvec = _mm_loadu_si128(pin+k); - accumulator = _mm_or_si128(accumulator,Delta(newvec , oldvec)); - oldvec = newvec; - } - initoffset = oldvec; - return maxbitas32int(accumulator); -} - - -/* maxbit over |length| integers with provided initial value */ -uint32_t simdmaxbitsd1_length(uint32_t initvalue, const uint32_t * in, - uint32_t length) { - __m128i newvec; - __m128i oldvec; - __m128i initoffset; - __m128i accumulator; - const __m128i *pin; - uint32_t tmparray[4]; - uint32_t k = 1; - uint32_t acc; - - assert(length > 0); - - pin = (const __m128i *)(in); - initoffset = _mm_set1_epi32(initvalue); - switch (length) { - case 1: - newvec = _mm_set1_epi32(in[0]); - break; - case 2: - newvec = _mm_setr_epi32(in[0], in[1], in[1], in[1]); - break; - case 3: - newvec = _mm_setr_epi32(in[0], in[1], in[2], in[2]); - break; - default: - newvec = _mm_loadu_si128(pin); - break; - } - accumulator = Delta(newvec, initoffset); - oldvec = newvec; - - /* process 4 integers and build an accumulator */ - while (k * 4 + 4 <= length) { - newvec = _mm_loadu_si128(pin + k); - accumulator = _mm_or_si128(accumulator, Delta(newvec, oldvec)); - oldvec = newvec; - k++; - } - - /* extract the accumulator as an integer */ - _mm_storeu_si128((__m128i *)(tmparray), accumulator); - acc = tmparray[0] | tmparray[1] | tmparray[2] | tmparray[3]; - - /* now process the remaining integers */ - for (k *= 4; k < length; k++) - acc |= in[k] - (k == 0 ? initvalue : in[k - 1]); - - /* return the number of bits */ - return bits(acc); -} diff --git a/cpp/simdcomp/src/simdfor.c b/cpp/simdcomp/src/simdfor.c deleted file mode 100644 index 67ddaa6b5..000000000 --- a/cpp/simdcomp/src/simdfor.c +++ /dev/null @@ -1,14501 +0,0 @@ -/** - * This code is released under a BSD License. - */ - -#include "simdfor.h" - - - -static __m128i iunpackFOR0(__m128i initOffset, const __m128i * _in , uint32_t * _out) { - __m128i *out = (__m128i*)(_out); - int i; - (void) _in; - for (i = 0; i < 8; ++i) { - _mm_store_si128(out++, initOffset); - _mm_store_si128(out++, initOffset); - _mm_store_si128(out++, initOffset); - _mm_store_si128(out++, initOffset); - } - - return initOffset; -} - - - - -static void ipackFOR0(__m128i initOffset , const uint32_t * _in , __m128i * out ) { - (void) initOffset; - (void) _in; - (void) out; -} - - -static void ipackFOR1(__m128i initOffset, const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - __m128i CurrIn = _mm_load_si128(in); - __m128i InReg = _mm_sub_epi32(CurrIn, initOffset); - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 1)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 3)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 5)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 7)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 9)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 11)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 13)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 15)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 17)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 19)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 21)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 23)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 25)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 27)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 29)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 31)); - _mm_store_si128(out, OutReg); - - -} - - - -static void ipackFOR2(__m128i initOffset, const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - __m128i CurrIn = _mm_load_si128(in); - __m128i InReg = _mm_sub_epi32(CurrIn, initOffset); - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_store_si128(out, OutReg); - - -} - - - -static void ipackFOR3(__m128i initOffset, const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - __m128i CurrIn = _mm_load_si128(in); - __m128i InReg = _mm_sub_epi32(CurrIn, initOffset); - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 3)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 9)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 15)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 21)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 27)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 3 - 1); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 1)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 7)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 13)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 19)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 25)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 31)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 3 - 2); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 5)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 11)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 17)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 23)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 29)); - _mm_store_si128(out, OutReg); - - -} - - - -static void ipackFOR4(__m128i initOffset, const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - __m128i CurrIn = _mm_load_si128(in); - __m128i InReg = _mm_sub_epi32(CurrIn, initOffset); - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - -} - - - -static void ipackFOR5(__m128i initOffset, const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - __m128i CurrIn = _mm_load_si128(in); - __m128i InReg = _mm_sub_epi32(CurrIn, initOffset); - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 5)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 15)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 25)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 5 - 3); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 3)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 13)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 23)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 5 - 1); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 1)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 11)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 21)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 31)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 5 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 9)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 19)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 29)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 5 - 2); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 7)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 17)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 27)); - _mm_store_si128(out, OutReg); - - -} - - - -static void ipackFOR6(__m128i initOffset, const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - __m128i CurrIn = _mm_load_si128(in); - __m128i InReg = _mm_sub_epi32(CurrIn, initOffset); - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 6 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 6 - 2); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 6 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 6 - 2); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_store_si128(out, OutReg); - - -} - - - -static void ipackFOR7(__m128i initOffset, const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - __m128i CurrIn = _mm_load_si128(in); - __m128i InReg = _mm_sub_epi32(CurrIn, initOffset); - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 7)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 21)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 7 - 3); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 3)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 17)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 31)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 7 - 6); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 13)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 27)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 7 - 2); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 9)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 23)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 7 - 5); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 5)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 19)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 7 - 1); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 1)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 15)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 29)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 7 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 11)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 25)); - _mm_store_si128(out, OutReg); - - -} - - - -static void ipackFOR8(__m128i initOffset, const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - __m128i CurrIn = _mm_load_si128(in); - __m128i InReg = _mm_sub_epi32(CurrIn, initOffset); - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - -} - - - -static void ipackFOR9(__m128i initOffset, const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - __m128i CurrIn = _mm_load_si128(in); - __m128i InReg = _mm_sub_epi32(CurrIn, initOffset); - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 9)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 27)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 9 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 13)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 31)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 9 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 17)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 9 - 3); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 3)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 21)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 9 - 7); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 7)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 25)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 9 - 2); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 11)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 29)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 9 - 6); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 15)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 9 - 1); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 1)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 19)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 9 - 5); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 5)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 23)); - _mm_store_si128(out, OutReg); - - -} - - - -static void ipackFOR10(__m128i initOffset, const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - __m128i CurrIn = _mm_load_si128(in); - __m128i InReg = _mm_sub_epi32(CurrIn, initOffset); - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 10 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 10 - 6); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 10 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 10 - 2); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 10 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 10 - 6); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 10 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 10 - 2); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_store_si128(out, OutReg); - - -} - - - -static void ipackFOR11(__m128i initOffset, const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - __m128i CurrIn = _mm_load_si128(in); - __m128i InReg = _mm_sub_epi32(CurrIn, initOffset); - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 11)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 11 - 1); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 1)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 23)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 11 - 2); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 13)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 11 - 3); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 3)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 25)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 11 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 15)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 11 - 5); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 5)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 27)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 11 - 6); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 17)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 11 - 7); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 7)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 29)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 11 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 19)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 11 - 9); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 9)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 31)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 11 - 10); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 21)); - _mm_store_si128(out, OutReg); - - -} - - - -static void ipackFOR12(__m128i initOffset, const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - __m128i CurrIn = _mm_load_si128(in); - __m128i InReg = _mm_sub_epi32(CurrIn, initOffset); - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 12 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 12 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 12 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 12 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 12 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 12 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 12 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 12 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_store_si128(out, OutReg); - - -} - - - -static void ipackFOR13(__m128i initOffset, const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - __m128i CurrIn = _mm_load_si128(in); - __m128i InReg = _mm_sub_epi32(CurrIn, initOffset); - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 13)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 13 - 7); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 7)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 13 - 1); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 1)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 27)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 13 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 21)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 13 - 2); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 15)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 13 - 9); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 9)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 13 - 3); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 3)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 29)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 13 - 10); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 23)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 13 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 17)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 13 - 11); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 11)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 13 - 5); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 5)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 31)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 13 - 12); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 25)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 13 - 6); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 19)); - _mm_store_si128(out, OutReg); - - -} - - - -static void ipackFOR14(__m128i initOffset, const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - __m128i CurrIn = _mm_load_si128(in); - __m128i InReg = _mm_sub_epi32(CurrIn, initOffset); - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 14 - 10); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 14 - 6); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 14 - 2); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 14 - 12); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 14 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 14 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 14 - 10); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 14 - 6); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 14 - 2); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 14 - 12); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 14 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 14 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_store_si128(out, OutReg); - - -} - - - -static void ipackFOR15(__m128i initOffset, const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - __m128i CurrIn = _mm_load_si128(in); - __m128i InReg = _mm_sub_epi32(CurrIn, initOffset); - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 15)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 13); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 13)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 11); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 11)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 9); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 9)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 7); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 7)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 5); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 5)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 3); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 3)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 1); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 1)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 31)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 14); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 29)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 12); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 27)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 10); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 25)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 23)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 6); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 21)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 19)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 15 - 2); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 17)); - _mm_store_si128(out, OutReg); - - -} - - - -static void ipackFOR16(__m128i initOffset, const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - __m128i CurrIn = _mm_load_si128(in); - __m128i InReg = _mm_sub_epi32(CurrIn, initOffset); - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - -} - - - -static void ipackFOR17(__m128i initOffset, const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - __m128i CurrIn = _mm_load_si128(in); - __m128i InReg = _mm_sub_epi32(CurrIn, initOffset); - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 17)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 2); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 19)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 21)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 6); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 23)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 25)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 10); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 27)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 12); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 29)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 14); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 31)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 16); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 1); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 1)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 3); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 3)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 5); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 5)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 7); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 7)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 9); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 9)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 11); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 11)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 13); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 13)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 17 - 15); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 15)); - _mm_store_si128(out, OutReg); - - -} - - - -static void ipackFOR18(__m128i initOffset, const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - __m128i CurrIn = _mm_load_si128(in); - __m128i InReg = _mm_sub_epi32(CurrIn, initOffset); - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 12); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 16); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 2); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 6); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 10); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 14); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 12); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 16); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 2); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 6); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 10); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 18 - 14); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - _mm_store_si128(out, OutReg); - - -} - - - -static void ipackFOR19(__m128i initOffset, const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - __m128i CurrIn = _mm_load_si128(in); - __m128i InReg = _mm_sub_epi32(CurrIn, initOffset); - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 19)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 6); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 25)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 12); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 31)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 18); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 5); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 5)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 11); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 11)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 17); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 17)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 23)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 10); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 29)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 16); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 3); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 3)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 9); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 9)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 15); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 15)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 2); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 21)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 27)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 14); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 1); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 1)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 7); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 7)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 19 - 13); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 13)); - _mm_store_si128(out, OutReg); - - -} - - - -static void ipackFOR20(__m128i initOffset, const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - __m128i CurrIn = _mm_load_si128(in); - __m128i InReg = _mm_sub_epi32(CurrIn, initOffset); - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 16); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 12); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 16); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 12); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 16); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 12); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 16); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 20 - 12); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_store_si128(out, OutReg); - - -} - - - -static void ipackFOR21(__m128i initOffset, const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - __m128i CurrIn = _mm_load_si128(in); - __m128i InReg = _mm_sub_epi32(CurrIn, initOffset); - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 21)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 10); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 31)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 20); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 9); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 9)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 19); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 19)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 29)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 18); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 7); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 7)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 17); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 17)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 6); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 27)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 16); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 5); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 5)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 15); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 15)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 25)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 14); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 3); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 3)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 13); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 13)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 2); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 23)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 12); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 1); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 1)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 21 - 11); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 11)); - _mm_store_si128(out, OutReg); - - -} - - - -static void ipackFOR22(__m128i initOffset, const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - __m128i CurrIn = _mm_load_si128(in); - __m128i InReg = _mm_sub_epi32(CurrIn, initOffset); - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 12); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 2); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 14); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 16); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 6); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 18); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 20); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 10); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 12); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 2); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 14); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 16); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 6); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 18); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 20); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 22 - 10); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - _mm_store_si128(out, OutReg); - - -} - - - -static void ipackFOR23(__m128i initOffset, const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - __m128i CurrIn = _mm_load_si128(in); - __m128i InReg = _mm_sub_epi32(CurrIn, initOffset); - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 23)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 14); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 5); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 5)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 19); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 19)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 10); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 1); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 1)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 15); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 15)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 6); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 29)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 20); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 11); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 11)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 2); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 25)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 16); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 7); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 7)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 21); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 21)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 12); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 3); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 3)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 17); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 17)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 31)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 22); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 13); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 13)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 27)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 18); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 23 - 9); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 9)); - _mm_store_si128(out, OutReg); - - -} - - - -static void ipackFOR24(__m128i initOffset, const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - __m128i CurrIn = _mm_load_si128(in); - __m128i InReg = _mm_sub_epi32(CurrIn, initOffset); - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 16); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 16); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 16); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 16); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 16); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 16); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 16); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 16); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 24 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_store_si128(out, OutReg); - - -} - - - -static void ipackFOR25(__m128i initOffset, const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - __m128i CurrIn = _mm_load_si128(in); - __m128i InReg = _mm_sub_epi32(CurrIn, initOffset); - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 25)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 18); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 11); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 11)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 29)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 22); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 15); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 15)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 1); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 1)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 19); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 19)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 12); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 5); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 5)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 23); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 23)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 16); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 9); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 9)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 2); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 27)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 20); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 13); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 13)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 6); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 31)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 24); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 17); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 17)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 10); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 3); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 3)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 21); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 21)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 14); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 25 - 7); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 7)); - _mm_store_si128(out, OutReg); - - -} - - - -static void ipackFOR26(__m128i initOffset, const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - __m128i CurrIn = _mm_load_si128(in); - __m128i InReg = _mm_sub_epi32(CurrIn, initOffset); - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 20); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 14); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 2); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 22); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 16); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 10); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 24); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 18); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 12); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 6); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 20); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 14); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 2); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 22); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 16); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 10); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 24); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 18); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 12); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 26 - 6); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - _mm_store_si128(out, OutReg); - - -} - - - -static void ipackFOR27(__m128i initOffset, const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - __m128i CurrIn = _mm_load_si128(in); - __m128i InReg = _mm_sub_epi32(CurrIn, initOffset); - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 27)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 22); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 17); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 17)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 12); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 7); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 7)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 2); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 29)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 24); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 19); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 19)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 14); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 9); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 9)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 31)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 26); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 21); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 21)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 16); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 11); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 11)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 6); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 1); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 1)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 23); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 23)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 18); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 13); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 13)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 3); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 3)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 25); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 25)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 20); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 15); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 15)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 10); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 27 - 5); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 5)); - _mm_store_si128(out, OutReg); - - -} - - - -static void ipackFOR28(__m128i initOffset, const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - __m128i CurrIn = _mm_load_si128(in); - __m128i InReg = _mm_sub_epi32(CurrIn, initOffset); - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 24); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 20); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 16); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 12); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 24); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 20); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 16); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 12); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 24); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 20); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 16); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 12); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 24); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 20); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 16); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 12); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 28 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - _mm_store_si128(out, OutReg); - - -} - - - -static void ipackFOR29(__m128i initOffset, const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - __m128i CurrIn = _mm_load_si128(in); - __m128i InReg = _mm_sub_epi32(CurrIn, initOffset); - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 29)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 26); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 23); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 23)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 20); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 17); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 17)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 14); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 11); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 11)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 5); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 5)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 2); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 31)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 28); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 25); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 25)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 22); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 19); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 19)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 16); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 13); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 13)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 10); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 7); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 7)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 1); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 1)); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 27); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 27)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 24); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 21); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 21)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 18); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 15); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 15)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 12); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 9); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 9)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 6); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 29 - 3); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 3)); - _mm_store_si128(out, OutReg); - - -} - - - -static void ipackFOR30(__m128i initOffset, const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - __m128i CurrIn = _mm_load_si128(in); - __m128i InReg = _mm_sub_epi32(CurrIn, initOffset); - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 28); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 26); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 24); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 22); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 20); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 18); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 16); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 14); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 12); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 10); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 6); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 2); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - _mm_store_si128(out, OutReg); - - ++out; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 28); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 26); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 24); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 22); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 20); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 18); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 16); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 14); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 12); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 10); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 6); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 30 - 2); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - _mm_store_si128(out, OutReg); - - -} - - - -static void ipackFOR31(__m128i initOffset, const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - - __m128i CurrIn = _mm_load_si128(in); - __m128i InReg = _mm_sub_epi32(CurrIn, initOffset); - OutReg = InReg; - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 31)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 30); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 30)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 29); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 29)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 28); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 28)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 27); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 27)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 26); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 26)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 25); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 25)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 24); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 24)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 23); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 23)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 22); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 22)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 21); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 21)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 20); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 20)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 19); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 19)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 18); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 18)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 17); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 17)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 16); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 16)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 15); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 15)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 14); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 14)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 13); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 13)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 12); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 12)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 11); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 11)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 10); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 10)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 9); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 9)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 8); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 8)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 7); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 7)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 6); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 6)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 5); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 5)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 4); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 4)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 3); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 3)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 2); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 2)); - _mm_store_si128(out, OutReg); - - ++out; - OutReg = _mm_srli_epi32(InReg, 31 - 1); - ++in; - CurrIn = _mm_load_si128(in); - InReg = _mm_sub_epi32(CurrIn, initOffset); - - OutReg = _mm_or_si128(OutReg, _mm_slli_epi32(InReg, 1)); - _mm_store_si128(out, OutReg); - - -} - - - -static void ipackFOR32(__m128i initOffset, const uint32_t * _in, __m128i * out) { - const __m128i *in = (const __m128i*)(_in); - __m128i OutReg; - - __m128i InReg = _mm_load_si128(in); - (void) initOffset; - - OutReg = InReg; - _mm_store_si128(out, OutReg); - - ++out; - ++in; - InReg = _mm_load_si128(in); - - OutReg = InReg; - _mm_store_si128(out, OutReg); - - ++out; - ++in; - InReg = _mm_load_si128(in); - - OutReg = InReg; - _mm_store_si128(out, OutReg); - - ++out; - ++in; - InReg = _mm_load_si128(in); - - OutReg = InReg; - _mm_store_si128(out, OutReg); - - ++out; - ++in; - InReg = _mm_load_si128(in); - - OutReg = InReg; - _mm_store_si128(out, OutReg); - - ++out; - ++in; - InReg = _mm_load_si128(in); - - OutReg = InReg; - _mm_store_si128(out, OutReg); - - ++out; - ++in; - InReg = _mm_load_si128(in); - - OutReg = InReg; - _mm_store_si128(out, OutReg); - - ++out; - ++in; - InReg = _mm_load_si128(in); - - OutReg = InReg; - _mm_store_si128(out, OutReg); - - ++out; - ++in; - InReg = _mm_load_si128(in); - - OutReg = InReg; - _mm_store_si128(out, OutReg); - - ++out; - ++in; - InReg = _mm_load_si128(in); - - OutReg = InReg; - _mm_store_si128(out, OutReg); - - ++out; - ++in; - InReg = _mm_load_si128(in); - - OutReg = InReg; - _mm_store_si128(out, OutReg); - - ++out; - ++in; - InReg = _mm_load_si128(in); - - OutReg = InReg; - _mm_store_si128(out, OutReg); - - ++out; - ++in; - InReg = _mm_load_si128(in); - - OutReg = InReg; - _mm_store_si128(out, OutReg); - - ++out; - ++in; - InReg = _mm_load_si128(in); - - OutReg = InReg; - _mm_store_si128(out, OutReg); - - ++out; - ++in; - InReg = _mm_load_si128(in); - - OutReg = InReg; - _mm_store_si128(out, OutReg); - - ++out; - ++in; - InReg = _mm_load_si128(in); - - OutReg = InReg; - _mm_store_si128(out, OutReg); - - ++out; - ++in; - InReg = _mm_load_si128(in); - - OutReg = InReg; - _mm_store_si128(out, OutReg); - - ++out; - ++in; - InReg = _mm_load_si128(in); - - OutReg = InReg; - _mm_store_si128(out, OutReg); - - ++out; - ++in; - InReg = _mm_load_si128(in); - - OutReg = InReg; - _mm_store_si128(out, OutReg); - - ++out; - ++in; - InReg = _mm_load_si128(in); - - OutReg = InReg; - _mm_store_si128(out, OutReg); - - ++out; - ++in; - InReg = _mm_load_si128(in); - - OutReg = InReg; - _mm_store_si128(out, OutReg); - - ++out; - ++in; - InReg = _mm_load_si128(in); - - OutReg = InReg; - _mm_store_si128(out, OutReg); - - ++out; - ++in; - InReg = _mm_load_si128(in); - - OutReg = InReg; - _mm_store_si128(out, OutReg); - - ++out; - ++in; - InReg = _mm_load_si128(in); - - OutReg = InReg; - _mm_store_si128(out, OutReg); - - ++out; - ++in; - InReg = _mm_load_si128(in); - - OutReg = InReg; - _mm_store_si128(out, OutReg); - - ++out; - ++in; - InReg = _mm_load_si128(in); - - OutReg = InReg; - _mm_store_si128(out, OutReg); - - ++out; - ++in; - InReg = _mm_load_si128(in); - - OutReg = InReg; - _mm_store_si128(out, OutReg); - - ++out; - ++in; - InReg = _mm_load_si128(in); - - OutReg = InReg; - _mm_store_si128(out, OutReg); - - ++out; - ++in; - InReg = _mm_load_si128(in); - - OutReg = InReg; - _mm_store_si128(out, OutReg); - - ++out; - ++in; - InReg = _mm_load_si128(in); - - OutReg = InReg; - _mm_store_si128(out, OutReg); - - ++out; - ++in; - InReg = _mm_load_si128(in); - - OutReg = InReg; - _mm_store_si128(out, OutReg); - - ++out; - ++in; - InReg = _mm_load_si128(in); - - OutReg = InReg; - _mm_store_si128(out, OutReg); - - -} - - - - -static __m128i iunpackFOR1(__m128i initOffset, const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_load_si128(in); - __m128i OutReg; - __m128i tmp; - const __m128i mask = _mm_set1_epi32((1U<<1)-1); - - - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,1); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,2); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,3); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,5); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,6); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,7); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,9); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,10); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,11); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,13); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,14); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,15); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,17); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,18); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,19); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,21); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,22); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,23); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,25); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,26); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,27); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,29); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,30); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,31); - OutReg = tmp; - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - - return initOffset; - -} - - - - -static __m128i iunpackFOR2(__m128i initOffset, const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_load_si128(in); - __m128i OutReg; - __m128i tmp; - const __m128i mask = _mm_set1_epi32((1U<<2)-1); - - - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,2); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,6); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,10); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,14); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,18); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,22); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,26); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,30); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,2); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,6); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,10); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,14); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,18); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,22); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,26); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,30); - OutReg = tmp; - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - - return initOffset; - -} - - - - -static __m128i iunpackFOR3(__m128i initOffset, const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_load_si128(in); - __m128i OutReg; - __m128i tmp; - const __m128i mask = _mm_set1_epi32((1U<<3)-1); - - - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,3); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,6); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,9); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,15); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,18); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,21); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,27); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,30); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 3-1), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,1); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,7); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,10); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,13); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,19); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,22); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,25); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,31); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 3-2), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,2); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,5); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,11); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,14); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,17); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,23); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,26); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,29); - OutReg = tmp; - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - - return initOffset; - -} - - - - -static __m128i iunpackFOR4(__m128i initOffset, const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_load_si128(in); - __m128i OutReg; - __m128i tmp; - const __m128i mask = _mm_set1_epi32((1U<<4)-1); - - - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - - return initOffset; - -} - - - - -static __m128i iunpackFOR5(__m128i initOffset, const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_load_si128(in); - __m128i OutReg; - __m128i tmp; - const __m128i mask = _mm_set1_epi32((1U<<5)-1); - - - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,5); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,10); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,15); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,25); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,30); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 5-3), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,3); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,13); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,18); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,23); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 5-1), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,1); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,6); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,11); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,21); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,26); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,31); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 5-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,9); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,14); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,19); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,29); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 5-2), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,2); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,7); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,17); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,22); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,27); - OutReg = tmp; - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - - return initOffset; - -} - - - - -static __m128i iunpackFOR6(__m128i initOffset, const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_load_si128(in); - __m128i OutReg; - __m128i tmp; - const __m128i mask = _mm_set1_epi32((1U<<6)-1); - - - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,6); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,18); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,30); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 6-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,10); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,22); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 6-2), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,2); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,14); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,26); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,6); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,18); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,30); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 6-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,10); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,22); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 6-2), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,2); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,14); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,26); - OutReg = tmp; - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - - return initOffset; - -} - - - - -static __m128i iunpackFOR7(__m128i initOffset, const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_load_si128(in); - __m128i OutReg; - __m128i tmp; - const __m128i mask = _mm_set1_epi32((1U<<7)-1); - - - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,7); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,14); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,21); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 7-3), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,3); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,10); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,17); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,31); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 7-6), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,6); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,13); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,27); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 7-2), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,2); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,9); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,23); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,30); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 7-5), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,5); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,19); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,26); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 7-1), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,1); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,15); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,22); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,29); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 7-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,11); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,18); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,25); - OutReg = tmp; - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - - return initOffset; - -} - - - - -static __m128i iunpackFOR8(__m128i initOffset, const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_load_si128(in); - __m128i OutReg; - __m128i tmp; - const __m128i mask = _mm_set1_epi32((1U<<8)-1); - - - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - - return initOffset; - -} - - - - -static __m128i iunpackFOR9(__m128i initOffset, const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_load_si128(in); - __m128i OutReg; - __m128i tmp; - const __m128i mask = _mm_set1_epi32((1U<<9)-1); - - - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,9); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,18); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,27); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 9-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,13); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,22); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,31); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 9-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,17); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,26); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 9-3), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,3); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,21); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,30); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 9-7), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,7); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,25); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 9-2), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,2); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,11); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,29); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 9-6), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,6); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,15); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 9-1), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,1); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,10); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,19); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 9-5), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,5); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,14); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,23); - OutReg = tmp; - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - - return initOffset; - -} - - - - -static __m128i iunpackFOR10(__m128i initOffset, const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_load_si128(in); - __m128i OutReg; - __m128i tmp; - const __m128i mask = _mm_set1_epi32((1U<<10)-1); - - - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,10); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,30); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 10-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,18); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 10-6), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,6); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,26); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 10-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,14); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 10-2), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,2); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,22); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,10); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,30); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 10-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,18); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 10-6), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,6); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,26); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 10-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,14); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 10-2), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,2); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,22); - OutReg = tmp; - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - - return initOffset; - -} - - - - -static __m128i iunpackFOR11(__m128i initOffset, const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_load_si128(in); - __m128i OutReg; - __m128i tmp; - const __m128i mask = _mm_set1_epi32((1U<<11)-1); - - - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,11); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,22); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 11-1), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,1); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,23); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 11-2), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,2); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,13); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 11-3), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,3); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,14); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,25); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 11-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,15); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,26); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 11-5), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,5); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,27); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 11-6), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,6); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,17); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 11-7), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,7); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,18); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,29); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 11-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,19); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,30); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 11-9), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,9); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,31); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 11-10), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,10); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,21); - OutReg = tmp; - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - - return initOffset; - -} - - - - -static __m128i iunpackFOR12(__m128i initOffset, const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_load_si128(in); - __m128i OutReg; - __m128i tmp; - const __m128i mask = _mm_set1_epi32((1U<<12)-1); - - - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 12-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 12-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 12-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 12-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 12-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 12-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 12-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 12-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = tmp; - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - - return initOffset; - -} - - - - -static __m128i iunpackFOR13(__m128i initOffset, const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_load_si128(in); - __m128i OutReg; - __m128i tmp; - const __m128i mask = _mm_set1_epi32((1U<<13)-1); - - - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,13); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,26); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 13-7), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,7); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 13-1), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,1); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,14); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,27); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 13-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,21); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 13-2), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,2); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,15); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 13-9), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,9); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,22); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 13-3), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,3); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,29); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 13-10), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,10); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,23); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 13-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,17); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,30); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 13-11), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,11); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 13-5), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,5); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,18); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,31); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 13-12), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,25); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 13-6), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,6); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,19); - OutReg = tmp; - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - - return initOffset; - -} - - - - -static __m128i iunpackFOR14(__m128i initOffset, const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_load_si128(in); - __m128i OutReg; - __m128i tmp; - const __m128i mask = _mm_set1_epi32((1U<<14)-1); - - - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,14); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 14-10), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,10); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 14-6), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,6); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 14-2), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,2); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,30); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 14-12), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,26); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 14-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,22); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 14-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,18); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,14); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 14-10), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,10); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 14-6), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,6); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 14-2), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,2); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,30); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 14-12), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,26); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 14-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,22); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 14-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,18); - OutReg = tmp; - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - - return initOffset; - -} - - - - -static __m128i iunpackFOR15(__m128i initOffset, const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_load_si128(in); - __m128i OutReg; - __m128i tmp; - const __m128i mask = _mm_set1_epi32((1U<<15)-1); - - - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,15); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,30); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 15-13), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,13); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 15-11), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,11); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,26); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 15-9), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,9); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 15-7), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,7); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,22); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 15-5), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,5); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 15-3), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,3); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,18); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 15-1), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,1); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,31); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 15-14), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,14); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,29); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 15-12), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,27); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 15-10), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,10); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,25); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 15-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,23); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 15-6), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,6); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,21); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 15-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,19); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 15-2), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,2); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,17); - OutReg = tmp; - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - - return initOffset; - -} - - - - -static __m128i iunpackFOR16(__m128i initOffset, const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_load_si128(in); - __m128i OutReg; - __m128i tmp; - const __m128i mask = _mm_set1_epi32((1U<<16)-1); - - - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - - return initOffset; - -} - - - - -static __m128i iunpackFOR17(__m128i initOffset, const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_load_si128(in); - __m128i OutReg; - __m128i tmp; - const __m128i mask = _mm_set1_epi32((1U<<17)-1); - - - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,17); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 17-2), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,2); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,19); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 17-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,21); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 17-6), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,6); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,23); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 17-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,25); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 17-10), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,10); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,27); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 17-12), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,29); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 17-14), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,14); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,31); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 17-16), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 17-1), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,1); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,18); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 17-3), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,3); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 17-5), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,5); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,22); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 17-7), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,7); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 17-9), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,9); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,26); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 17-11), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,11); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 17-13), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,13); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,30); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 17-15), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,15); - OutReg = tmp; - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - - return initOffset; - -} - - - - -static __m128i iunpackFOR18(__m128i initOffset, const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_load_si128(in); - __m128i OutReg; - __m128i tmp; - const __m128i mask = _mm_set1_epi32((1U<<18)-1); - - - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,18); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 18-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,22); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 18-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,26); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 18-12), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,30); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 18-16), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 18-2), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,2); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 18-6), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,6); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 18-10), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,10); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 18-14), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,14); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,18); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 18-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,22); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 18-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,26); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 18-12), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,30); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 18-16), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 18-2), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,2); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 18-6), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,6); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 18-10), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,10); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 18-14), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,14); - OutReg = tmp; - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - - return initOffset; - -} - - - - -static __m128i iunpackFOR19(__m128i initOffset, const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_load_si128(in); - __m128i OutReg; - __m128i tmp; - const __m128i mask = _mm_set1_epi32((1U<<19)-1); - - - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,19); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 19-6), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,6); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,25); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 19-12), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,31); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 19-18), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,18); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 19-5), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,5); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 19-11), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,11); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,30); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 19-17), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,17); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 19-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,23); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 19-10), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,10); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,29); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 19-16), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 19-3), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,3); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,22); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 19-9), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,9); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 19-15), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,15); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 19-2), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,2); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,21); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 19-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,27); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 19-14), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,14); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 19-1), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,1); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 19-7), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,7); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,26); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 19-13), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,13); - OutReg = tmp; - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - - return initOffset; - -} - - - - -static __m128i iunpackFOR20(__m128i initOffset, const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_load_si128(in); - __m128i OutReg; - __m128i tmp; - const __m128i mask = _mm_set1_epi32((1U<<20)-1); - - - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 20-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 20-16), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 20-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 20-12), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 20-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 20-16), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 20-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 20-12), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 20-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 20-16), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 20-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 20-12), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 20-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 20-16), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 20-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 20-12), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = tmp; - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - - return initOffset; - -} - - - - -static __m128i iunpackFOR21(__m128i initOffset, const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_load_si128(in); - __m128i OutReg; - __m128i tmp; - const __m128i mask = _mm_set1_epi32((1U<<21)-1); - - - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,21); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-10), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,10); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,31); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-20), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-9), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,9); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,30); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-19), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,19); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,29); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-18), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,18); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-7), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,7); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-17), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,17); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-6), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,6); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,27); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-16), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-5), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,5); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,26); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-15), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,15); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,25); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-14), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,14); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-3), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,3); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-13), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,13); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-2), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,2); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,23); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-12), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-1), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,1); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,22); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 21-11), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,11); - OutReg = tmp; - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - - return initOffset; - -} - - - - -static __m128i iunpackFOR22(__m128i initOffset, const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_load_si128(in); - __m128i OutReg; - __m128i tmp; - const __m128i mask = _mm_set1_epi32((1U<<22)-1); - - - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,22); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-12), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-2), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,2); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-14), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,14); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,26); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-16), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-6), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,6); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-18), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,18); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,30); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-20), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-10), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,10); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,22); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-12), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-2), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,2); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-14), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,14); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,26); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-16), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-6), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,6); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-18), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,18); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,30); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-20), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 22-10), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,10); - OutReg = tmp; - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - - return initOffset; - -} - - - - -static __m128i iunpackFOR23(__m128i initOffset, const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_load_si128(in); - __m128i OutReg; - __m128i tmp; - const __m128i mask = _mm_set1_epi32((1U<<23)-1); - - - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,23); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-14), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,14); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-5), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,5); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-19), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,19); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-10), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,10); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-1), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,1); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-15), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,15); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-6), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,6); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,29); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-20), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-11), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,11); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-2), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,2); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,25); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-16), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-7), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,7); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,30); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-21), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,21); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-12), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-3), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,3); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,26); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-17), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,17); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,31); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-22), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,22); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-13), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,13); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,27); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-18), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,18); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 23-9), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,9); - OutReg = tmp; - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - - return initOffset; - -} - - - - -static __m128i iunpackFOR24(__m128i initOffset, const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_load_si128(in); - __m128i OutReg; - __m128i tmp; - const __m128i mask = _mm_set1_epi32((1U<<24)-1); - - - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = tmp; - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - - return initOffset; - -} - - - - -static __m128i iunpackFOR25(__m128i initOffset, const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_load_si128(in); - __m128i OutReg; - __m128i tmp; - const __m128i mask = _mm_set1_epi32((1U<<25)-1); - - - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,25); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-18), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,18); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-11), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,11); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,29); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-22), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,22); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-15), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,15); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-1), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,1); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,26); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-19), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,19); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-12), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-5), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,5); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,30); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-23), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,23); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-16), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-9), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,9); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-2), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,2); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,27); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-20), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-13), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,13); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-6), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,6); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,31); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-24), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-17), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,17); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-10), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,10); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-3), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,3); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-21), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,21); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-14), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,14); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 25-7), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,7); - OutReg = tmp; - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - - return initOffset; - -} - - - - -static __m128i iunpackFOR26(__m128i initOffset, const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_load_si128(in); - __m128i OutReg; - __m128i tmp; - const __m128i mask = _mm_set1_epi32((1U<<26)-1); - - - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,26); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-20), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-14), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,14); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-2), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,2); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-22), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,22); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-16), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-10), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,10); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,30); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-24), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-18), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,18); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-12), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-6), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,6); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,26); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-20), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-14), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,14); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-2), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,2); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-22), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,22); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-16), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-10), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,10); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,30); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-24), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-18), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,18); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-12), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 26-6), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,6); - OutReg = tmp; - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - - return initOffset; - -} - - - - -static __m128i iunpackFOR27(__m128i initOffset, const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_load_si128(in); - __m128i OutReg; - __m128i tmp; - const __m128i mask = _mm_set1_epi32((1U<<27)-1); - - - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,27); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-22), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,22); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-17), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,17); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-12), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-7), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,7); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-2), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,2); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,29); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-24), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-19), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,19); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-14), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,14); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-9), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,9); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,31); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-26), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,26); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-21), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,21); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-16), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-11), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,11); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-6), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,6); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-1), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,1); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-23), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,23); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-18), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,18); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-13), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,13); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-3), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,3); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,30); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-25), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,25); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-20), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-15), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,15); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-10), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,10); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 27-5), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,5); - OutReg = tmp; - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - - return initOffset; - -} - - - - -static __m128i iunpackFOR28(__m128i initOffset, const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_load_si128(in); - __m128i OutReg; - __m128i tmp; - const __m128i mask = _mm_set1_epi32((1U<<28)-1); - - - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-24), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-20), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-16), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-12), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-24), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-20), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-16), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-12), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-24), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-20), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-16), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-12), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-24), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-20), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-16), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-12), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 28-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = tmp; - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - - return initOffset; - -} - - - - -static __m128i iunpackFOR29(__m128i initOffset, const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_load_si128(in); - __m128i OutReg; - __m128i tmp; - const __m128i mask = _mm_set1_epi32((1U<<29)-1); - - - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,29); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-26), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,26); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-23), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,23); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-20), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-17), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,17); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-14), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,14); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-11), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,11); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-5), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,5); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-2), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,2); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,31); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-28), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-25), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,25); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-22), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,22); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-19), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,19); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-16), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-13), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,13); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-10), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,10); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-7), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,7); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-1), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,1); - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,30); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-27), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,27); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-24), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-21), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,21); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-18), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,18); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-15), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,15); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-12), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-9), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,9); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-6), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,6); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 29-3), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,3); - OutReg = tmp; - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - - return initOffset; - -} - - - - -static __m128i iunpackFOR30(__m128i initOffset, const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_load_si128(in); - __m128i OutReg; - __m128i tmp; - const __m128i mask = _mm_set1_epi32((1U<<30)-1); - - - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,30); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-28), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-26), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,26); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-24), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-22), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,22); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-20), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-18), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,18); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-16), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-14), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,14); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-12), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-10), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,10); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-6), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,6); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-2), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,2); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,30); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-28), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-26), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,26); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-24), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-22), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,22); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-20), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-18), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,18); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-16), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-14), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,14); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-12), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-10), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,10); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-6), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,6); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 30-2), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,2); - OutReg = tmp; - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - - return initOffset; - -} - - - - -static __m128i iunpackFOR31(__m128i initOffset, const __m128i* in, uint32_t * _out) { - - __m128i* out = (__m128i*)(_out); - __m128i InReg = _mm_load_si128(in); - __m128i OutReg; - __m128i tmp; - const __m128i mask = _mm_set1_epi32((1U<<31)-1); - - - - tmp = InReg; - OutReg = _mm_and_si128(tmp, mask); - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,31); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-30), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,30); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-29), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,29); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-28), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,28); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-27), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,27); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-26), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,26); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-25), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,25); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-24), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,24); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-23), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,23); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-22), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,22); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-21), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,21); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-20), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,20); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-19), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,19); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-18), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,18); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-17), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,17); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-16), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,16); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-15), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,15); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-14), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,14); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-13), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,13); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-12), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,12); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-11), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,11); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-10), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,10); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-9), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,9); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-8), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,8); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-7), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,7); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-6), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,6); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-5), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,5); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-4), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,4); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-3), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,3); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-2), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,2); - OutReg = tmp; - ++in; InReg = _mm_load_si128(in); - OutReg = _mm_or_si128(OutReg, _mm_and_si128(_mm_slli_epi32(InReg, 31-1), mask)); - - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - tmp = _mm_srli_epi32(InReg,1); - OutReg = tmp; - OutReg = _mm_add_epi32(OutReg, initOffset); - _mm_store_si128(out++, OutReg); - - - return initOffset; - -} - - - -static __m128i iunpackFOR32(__m128i initvalue , const __m128i* in, uint32_t * _out) { - __m128i * mout = (__m128i *)_out; - __m128i invec; - size_t k; - (void) initvalue; - for(k = 0; k < 128/4; ++k) { - invec = _mm_load_si128(in++); - _mm_store_si128(mout++, invec); - } - return invec; -} - - - - - -void simdpackFOR(uint32_t initvalue, const uint32_t * in, __m128i * out, const uint32_t bit) { - __m128i initOffset = _mm_set1_epi32 (initvalue); - switch(bit) { - case 0: ipackFOR0(initOffset,in,out); break; - - case 1: ipackFOR1(initOffset,in,out); break; - - case 2: ipackFOR2(initOffset,in,out); break; - - case 3: ipackFOR3(initOffset,in,out); break; - - case 4: ipackFOR4(initOffset,in,out); break; - - case 5: ipackFOR5(initOffset,in,out); break; - - case 6: ipackFOR6(initOffset,in,out); break; - - case 7: ipackFOR7(initOffset,in,out); break; - - case 8: ipackFOR8(initOffset,in,out); break; - - case 9: ipackFOR9(initOffset,in,out); break; - - case 10: ipackFOR10(initOffset,in,out); break; - - case 11: ipackFOR11(initOffset,in,out); break; - - case 12: ipackFOR12(initOffset,in,out); break; - - case 13: ipackFOR13(initOffset,in,out); break; - - case 14: ipackFOR14(initOffset,in,out); break; - - case 15: ipackFOR15(initOffset,in,out); break; - - case 16: ipackFOR16(initOffset,in,out); break; - - case 17: ipackFOR17(initOffset,in,out); break; - - case 18: ipackFOR18(initOffset,in,out); break; - - case 19: ipackFOR19(initOffset,in,out); break; - - case 20: ipackFOR20(initOffset,in,out); break; - - case 21: ipackFOR21(initOffset,in,out); break; - - case 22: ipackFOR22(initOffset,in,out); break; - - case 23: ipackFOR23(initOffset,in,out); break; - - case 24: ipackFOR24(initOffset,in,out); break; - - case 25: ipackFOR25(initOffset,in,out); break; - - case 26: ipackFOR26(initOffset,in,out); break; - - case 27: ipackFOR27(initOffset,in,out); break; - - case 28: ipackFOR28(initOffset,in,out); break; - - case 29: ipackFOR29(initOffset,in,out); break; - - case 30: ipackFOR30(initOffset,in,out); break; - - case 31: ipackFOR31(initOffset,in,out); break; - - case 32: ipackFOR32(initOffset,in,out); break; - - default: break; - } -} - - - - -void simdunpackFOR(uint32_t initvalue, const __m128i * in, uint32_t * out, const uint32_t bit) { - __m128i initOffset = _mm_set1_epi32 (initvalue); - switch(bit) { - case 0: iunpackFOR0(initOffset, in,out); break; - - case 1: iunpackFOR1(initOffset, in,out); break; - - case 2: iunpackFOR2(initOffset, in,out); break; - - case 3: iunpackFOR3(initOffset, in,out); break; - - case 4: iunpackFOR4(initOffset, in,out); break; - - case 5: iunpackFOR5(initOffset, in,out); break; - - case 6: iunpackFOR6(initOffset, in,out); break; - - case 7: iunpackFOR7(initOffset, in,out); break; - - case 8: iunpackFOR8(initOffset, in,out); break; - - case 9: iunpackFOR9(initOffset, in,out); break; - - case 10: iunpackFOR10(initOffset, in,out); break; - - case 11: iunpackFOR11(initOffset, in,out); break; - - case 12: iunpackFOR12(initOffset, in,out); break; - - case 13: iunpackFOR13(initOffset, in,out); break; - - case 14: iunpackFOR14(initOffset, in,out); break; - - case 15: iunpackFOR15(initOffset, in,out); break; - - case 16: iunpackFOR16(initOffset, in,out); break; - - case 17: iunpackFOR17(initOffset, in,out); break; - - case 18: iunpackFOR18(initOffset, in,out); break; - - case 19: iunpackFOR19(initOffset, in,out); break; - - case 20: iunpackFOR20(initOffset, in,out); break; - - case 21: iunpackFOR21(initOffset, in,out); break; - - case 22: iunpackFOR22(initOffset, in,out); break; - - case 23: iunpackFOR23(initOffset, in,out); break; - - case 24: iunpackFOR24(initOffset, in,out); break; - - case 25: iunpackFOR25(initOffset, in,out); break; - - case 26: iunpackFOR26(initOffset, in,out); break; - - case 27: iunpackFOR27(initOffset, in,out); break; - - case 28: iunpackFOR28(initOffset, in,out); break; - - case 29: iunpackFOR29(initOffset, in,out); break; - - case 30: iunpackFOR30(initOffset, in,out); break; - - case 31: iunpackFOR31(initOffset, in,out); break; - - case 32: iunpackFOR32(initOffset, in,out); break; - - default: break; - } -} - - -uint32_t simdselectFOR(uint32_t initvalue, const __m128i *in, uint32_t bit, - int slot) { - const uint32_t * pin = (const uint32_t *) in; - if( bit == 0) { - return initvalue; - } else if (bit == 32) { - /* silly special case */ - return pin[slot]; - } else { - const int lane = slot % 4; /* we have 4 interleaved lanes */ - const int bitsinlane = (slot / 4) * bit; /* how many bits in lane */ - const int firstwordinlane = bitsinlane / 32; - const int secondwordinlane = (bitsinlane + bit - 1) / 32; - const uint32_t firstpart = pin[4 * firstwordinlane + lane] - >> (bitsinlane % 32); - const uint32_t mask = (1 << bit) - 1; - if (firstwordinlane == secondwordinlane) { - /* easy common case*/ - return initvalue + (firstpart & mask); - } else { - /* harder case where we need to combine two words */ - const uint32_t secondpart = pin[4 * firstwordinlane + 4 + lane]; - const int usablebitsinfirstword = 32 - (bitsinlane % 32); - return initvalue - + ((firstpart | (secondpart << usablebitsinfirstword)) - & mask); - } - } - -} - - - - -int simdsearchwithlengthFOR(uint32_t initvalue, const __m128i *in, uint32_t bit, - int length, uint32_t key, uint32_t *presult) { - int count = length; - int begin = 0; - uint32_t val; - while (count > 0) { - int step = count / 2; - val = simdselectFOR(initvalue, in, bit, begin + step); - if (val < key) { - begin += step + 1; - count -= step + 1; - } else count = step; - } - *presult = simdselectFOR(initvalue, in, bit, begin); - return begin; -} - -int simdpackFOR_compressedbytes(int length, const uint32_t bit) { - if(bit == 0) return 0;/* nothing to do */ - if(bit == 32) { - return length * sizeof(uint32_t); - } - return (((length + 3 )/ 4) * bit + 31 ) / 32 * sizeof(__m128i); -} - -__m128i * simdpackFOR_length(uint32_t initvalue, const uint32_t * in, int length, __m128i * out, const uint32_t bit) { - int k; - int inwordpointer; - __m128i P; - uint32_t firstpass; - __m128i offset; - if(bit == 0) return out;/* nothing to do */ - if(bit == 32) { - memcpy(out,in,length*sizeof(uint32_t)); - return (__m128i *)((uint32_t *) out + length); - } - offset = _mm_set1_epi32(initvalue); - inwordpointer = 0; - P = _mm_setzero_si128(); - for(k = 0; k < length / 4 ; ++k) { - __m128i value = _mm_sub_epi32(_mm_loadu_si128(((const __m128i * ) in + k)),offset); - P = _mm_or_si128(P,_mm_slli_epi32(value, inwordpointer)); - firstpass = sizeof(uint32_t) * 8 - inwordpointer; - if(bit -#include "simdintegratedbitpacking.h" - - -SIMDCOMP_ALIGNED(16) static int8_t shuffle_mask_bytes[256] = { - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 4, 5, 6, 7, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 8, 9, 10, 11, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 1, 2, 3, 8, 9, 10, 11, 8, 9, 10, 11, 12, 13, 14, 15, - 4, 5, 6, 7, 8, 9, 10, 11, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 12, 13, 14, 15, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 1, 2, 3, 12, 13, 14, 15, 8, 9, 10, 11, 12, 13, 14, 15, - 4, 5, 6, 7, 12, 13, 14, 15, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 1, 2, 3, 4, 5, 6, 7, 12, 13, 14, 15, 12, 13, 14, 15, - 8, 9, 10, 11, 12, 13, 14, 15, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 1, 2, 3, 8, 9, 10, 11, 12, 13, 14, 15, 12, 13, 14, 15, - 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 12, 13, 14, 15, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - }; -static const __m128i *shuffle_mask = (__m128i *) shuffle_mask_bytes; - -/* should emulate std:lower_bound*/ -static int lower_bound(uint32_t * A, uint32_t key, int imin, int imax) -{ - int imid; - imax --; - while(imin + 1 < imax) { - imid = imin + ((imax - imin) / 2); - - if (A[imid] >= key) { - imax = imid; - } else if (A[imid] < key) { - imin = imid; - } - } - if(A[imin] >= key) return imin; - return imax; -} - - -#define PrefixSum(ret, curr, prev) do { \ - const __m128i _tmp1 = _mm_add_epi32(_mm_slli_si128(curr, 8), curr); \ - const __m128i _tmp2 = _mm_add_epi32(_mm_slli_si128(_tmp1, 4), _tmp1); \ - ret = _mm_add_epi32(_tmp2, _mm_shuffle_epi32(prev, 0xff)); \ - } while (0) - - - -/* perform a lower-bound search for |key| in |out|; the resulting uint32 -* is stored in |*presult|.*/ -#define CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult) \ - do { \ - __m128i tmpout = _mm_sub_epi32(out, conversion); \ - uint32_t mmask = _mm_movemask_ps(_mm_castsi128_ps(_mm_cmplt_epi32(tmpout, key4))); \ - if (mmask != 15) { \ - const __m128i p = _mm_shuffle_epi8(out, shuffle_mask[mmask ^ 15]); \ - int offset; \ - int remaining = length - i; \ - SIMDCOMP_CTZ(offset, mmask ^ 15); \ - *presult = _mm_cvtsi128_si32(p); \ - if (offset < remaining) \ - return (i + offset); \ - } \ - i += 4; \ - if (i >= length) { /* reached end of array? */ \ - *presult = key + 1; \ - return (length); \ - } \ - } while (0) - -static int -iunpacksearchwithlength0(__m128i initOffset , const __m128i * _in, int length, - uint32_t key, uint32_t *presult) -{ - if (length > 0) { - uint32_t repeatedvalue = (uint32_t) _mm_extract_epi32(initOffset, 3); - if (repeatedvalue >= key) { - *presult = repeatedvalue; - return 0; - } - } - (void) _in; - *presult = key + 1; - return (length); -} - -static int -iunpacksearchwithlength1(__m128i initOffset, const __m128i *in, int length, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<1)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,13); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,15); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,17); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,19); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,21); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,23); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,25); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,27); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,29); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearchwithlength2(__m128i initOffset, const __m128i *in, int length, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<2)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearchwithlength3(__m128i initOffset, const __m128i *in, int length, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<3)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,15); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,21); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,27); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 3-1), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,13); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,19); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,25); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 3-2), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,17); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,23); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearchwithlength4(__m128i initOffset, const __m128i *in, int length, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<4)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearchwithlength5(__m128i initOffset, const __m128i *in, int length, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<5)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,15); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,25); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 5-3), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,13); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,23); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 5-1), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,21); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 5-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,19); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 5-2), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,17); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearchwithlength6(__m128i initOffset, const __m128i *in, int length, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<6)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 6-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 6-2), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 6-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 6-2), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearchwithlength7(__m128i initOffset, const __m128i *in, int length, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<7)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,21); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 7-3), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,17); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 7-6), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,13); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 7-2), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,23); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 7-5), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,19); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 7-1), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,15); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 7-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearchwithlength8(__m128i initOffset, const __m128i *in, int length, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<8)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearchwithlength9(__m128i initOffset, const __m128i *in, int length, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<9)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 9-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,13); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 9-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,17); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 9-3), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,21); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 9-7), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 9-2), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 9-6), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,15); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 9-1), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,19); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 9-5), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearchwithlength10(__m128i initOffset, const __m128i *in, int length, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<10)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 10-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 10-6), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 10-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 10-2), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 10-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 10-6), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 10-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 10-2), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearchwithlength11(__m128i initOffset, const __m128i *in, int length, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<11)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-1), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-2), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,13); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-3), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,15); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-5), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-6), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,17); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-7), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,19); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-9), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-10), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearchwithlength12(__m128i initOffset, const __m128i *in, int length, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<12)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 12-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 12-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 12-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 12-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 12-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 12-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 12-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 12-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearchwithlength13(__m128i initOffset, const __m128i *in, int length, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<13)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,13); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-7), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-1), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-2), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,15); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-9), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-3), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-10), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,17); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-11), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-5), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-12), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-6), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearchwithlength14(__m128i initOffset, const __m128i *in, int length, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<14)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-10), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-6), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-2), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-12), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-10), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-6), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-2), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-12), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearchwithlength15(__m128i initOffset, const __m128i *in, int length, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<15)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,15); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-13), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,13); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-11), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-9), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-7), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-5), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-3), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-1), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-14), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-12), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-10), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-6), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-2), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,17); - out = tmp; - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearchwithlength16(__m128i initOffset, const __m128i *in, int length, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<16)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearchwithlength17(__m128i initOffset, const __m128i *in, int length, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<17)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,17); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-2), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-6), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-10), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-12), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-14), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-16), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-1), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-3), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-5), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-7), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-9), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-11), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-13), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,13); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-15), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,15); - out = tmp; - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearchwithlength18(__m128i initOffset, const __m128i *in, int length, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<18)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-12), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-16), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-2), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-6), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-10), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-14), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-12), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-16), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-2), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-6), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-10), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-14), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearchwithlength19(__m128i initOffset, const __m128i *in, int length, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<19)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-6), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-12), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-18), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-5), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-11), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-17), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,17); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-10), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-16), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-3), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-9), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-15), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,15); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-2), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-14), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-1), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-7), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-13), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,13); - out = tmp; - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearchwithlength20(__m128i initOffset, const __m128i *in, int length, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<20)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-16), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-12), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-16), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-12), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-16), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-12), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-16), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-12), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearchwithlength21(__m128i initOffset, const __m128i *in, int length, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<21)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-10), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-20), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-9), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-19), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-18), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-7), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-17), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,17); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-6), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-16), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-5), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-15), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,15); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-14), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-3), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-13), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,13); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-2), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-12), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-1), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-11), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,11); - out = tmp; - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearchwithlength22(__m128i initOffset, const __m128i *in, int length, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<22)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-12), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-2), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-14), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-16), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-6), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-18), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-20), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-10), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-12), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-2), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-14), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-16), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-6), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-18), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-20), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-10), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearchwithlength23(__m128i initOffset, const __m128i *in, int length, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<23)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-14), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-5), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-19), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-10), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-1), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-15), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,15); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-6), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-20), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-11), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,11); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-2), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-16), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-7), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-21), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-12), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-3), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-17), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,17); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-22), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-13), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,13); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-18), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-9), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,9); - out = tmp; - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearchwithlength24(__m128i initOffset, const __m128i *in, int length, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<24)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearchwithlength25(__m128i initOffset, const __m128i *in, int length, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<25)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-18), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-11), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,11); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-22), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-15), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,15); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-1), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-19), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-12), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-5), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-23), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-16), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-9), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,9); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-2), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-20), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-13), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,13); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-6), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-24), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-17), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,17); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-10), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-3), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-21), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-14), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-7), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,7); - out = tmp; - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearchwithlength26(__m128i initOffset, const __m128i *in, int length, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<26)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-20), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-14), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-2), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-22), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-16), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-10), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-24), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-18), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-12), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-6), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-20), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-14), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-2), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-22), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-16), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-10), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-24), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-18), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-12), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-6), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = tmp; - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearchwithlength27(__m128i initOffset, const __m128i *in, int length, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<27)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-22), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-17), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,17); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-12), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-7), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,7); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-2), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-24), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-19), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-14), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-9), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,9); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-26), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-21), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-16), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-11), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,11); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-6), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-1), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-23), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-18), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-13), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,13); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-3), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-25), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-20), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-15), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,15); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-10), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-5), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,5); - out = tmp; - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearchwithlength28(__m128i initOffset, const __m128i *in, int length, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<28)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-24), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-20), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-16), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-12), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-24), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-20), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-16), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-12), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-24), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-20), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-16), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-12), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-24), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-20), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-16), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-12), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = tmp; - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearchwithlength29(__m128i initOffset, const __m128i *in, int length, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<29)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-26), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-23), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-20), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-17), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,17); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-14), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-11), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,11); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-5), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,5); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-2), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-28), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-25), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-22), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-19), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-16), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-13), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,13); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-10), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-7), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,7); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-1), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-27), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-24), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-21), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-18), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-15), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,15); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-12), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-9), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,9); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-6), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-3), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,3); - out = tmp; - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearchwithlength30(__m128i initOffset, const __m128i *in, int length, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<30)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-28), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-26), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-24), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-22), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-20), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-18), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-16), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-14), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-12), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-10), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-6), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-2), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-28), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-26), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-24), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-22), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-20), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-18), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-16), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-14), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-12), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-10), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-6), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-2), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = tmp; - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearchwithlength31(__m128i initOffset, const __m128i *in, int length, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<31)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-30), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-29), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-28), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-27), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-26), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-25), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-24), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-23), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-22), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-21), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-20), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-19), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-18), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-17), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,17); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-16), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-15), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,15); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-14), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-13), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,13); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-12), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-11), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,11); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-10), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-9), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,9); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-8), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-7), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,7); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-6), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-5), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,5); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-4), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-3), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,3); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-2), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-1), mask)); - - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - tmp = _mm_srli_epi32(InReg,1); - out = tmp; - PrefixSum(out, out, initOffset); - initOffset = out; - CHECK_AND_INCREMENT_WITH_LENGTH(i, out, length, key, presult); - - *presult = key + 1; - return (128); -} - - -static int -iunpacksearchwithlength32(__m128i initOffset, const __m128i *in, int length, - uint32_t key, uint32_t *presult) -{ - uint32_t * in32 = (uint32_t *)in; - int answer = lower_bound(in32, key, 0, length); - if(in32[answer] < key) { - *presult = key + 1; - return (length); - } - (void) initOffset; - *presult = in32[answer]; - return answer; -} - - -int -simdsearchwithlengthd1(uint32_t initvalue, const __m128i *in, uint32_t bit, int length, - uint32_t key, uint32_t *presult) -{ - __m128i initOffset = _mm_set1_epi32 (initvalue); - switch (bit) { - case 0: return iunpacksearchwithlength0(initOffset, in, length, key, presult); - - case 1: return iunpacksearchwithlength1(initOffset, in, length, key, presult); - - case 2: return iunpacksearchwithlength2(initOffset, in, length, key, presult); - - case 3: return iunpacksearchwithlength3(initOffset, in, length, key, presult); - - case 4: return iunpacksearchwithlength4(initOffset, in, length, key, presult); - - case 5: return iunpacksearchwithlength5(initOffset, in, length, key, presult); - - case 6: return iunpacksearchwithlength6(initOffset, in, length, key, presult); - - case 7: return iunpacksearchwithlength7(initOffset, in, length, key, presult); - - case 8: return iunpacksearchwithlength8(initOffset, in, length, key, presult); - - case 9: return iunpacksearchwithlength9(initOffset, in, length, key, presult); - - case 10: return iunpacksearchwithlength10(initOffset, in, length, key, presult); - - case 11: return iunpacksearchwithlength11(initOffset, in, length, key, presult); - - case 12: return iunpacksearchwithlength12(initOffset, in, length, key, presult); - - case 13: return iunpacksearchwithlength13(initOffset, in, length, key, presult); - - case 14: return iunpacksearchwithlength14(initOffset, in, length, key, presult); - - case 15: return iunpacksearchwithlength15(initOffset, in, length, key, presult); - - case 16: return iunpacksearchwithlength16(initOffset, in, length, key, presult); - - case 17: return iunpacksearchwithlength17(initOffset, in, length, key, presult); - - case 18: return iunpacksearchwithlength18(initOffset, in, length, key, presult); - - case 19: return iunpacksearchwithlength19(initOffset, in, length, key, presult); - - case 20: return iunpacksearchwithlength20(initOffset, in, length, key, presult); - - case 21: return iunpacksearchwithlength21(initOffset, in, length, key, presult); - - case 22: return iunpacksearchwithlength22(initOffset, in, length, key, presult); - - case 23: return iunpacksearchwithlength23(initOffset, in, length, key, presult); - - case 24: return iunpacksearchwithlength24(initOffset, in, length, key, presult); - - case 25: return iunpacksearchwithlength25(initOffset, in, length, key, presult); - - case 26: return iunpacksearchwithlength26(initOffset, in, length, key, presult); - - case 27: return iunpacksearchwithlength27(initOffset, in, length, key, presult); - - case 28: return iunpacksearchwithlength28(initOffset, in, length, key, presult); - - case 29: return iunpacksearchwithlength29(initOffset, in, length, key, presult); - - case 30: return iunpacksearchwithlength30(initOffset, in, length, key, presult); - - case 31: return iunpacksearchwithlength31(initOffset, in, length, key, presult); - - case 32: return iunpacksearchwithlength32(initOffset, in, length, key, presult); - - default: break; - } - return (-1); -} - - - - - -/* perform a lower-bound search for |key| in |out|; the resulting uint32 -* is stored in |*presult|.*/ -#define CHECK_AND_INCREMENT(i, out, key, presult) \ - do { \ - __m128i tmpout = _mm_sub_epi32(out, conversion); \ - uint32_t mmask = _mm_movemask_ps(_mm_castsi128_ps(_mm_cmplt_epi32(tmpout, key4))); \ - if (mmask != 15) { \ - __m128i p = _mm_shuffle_epi8(out, shuffle_mask[mmask ^ 15]); \ - int offset; \ - SIMDCOMP_CTZ(offset, mmask ^ 15); \ - *presult = _mm_cvtsi128_si32(p); \ - return (i + offset); \ - } \ - i += 4; \ - } while (0) - -static int -iunpacksearch0(__m128i * initOffset , const __m128i * _in, - uint32_t key, uint32_t *presult) -{ - uint32_t repeatedvalue = (uint32_t) _mm_extract_epi32(*initOffset, 3); - if (repeatedvalue >= key) { - *presult = repeatedvalue; - return 0; - } - *presult = key + 1; - (void)_in; - return (128); -} - -static int -iunpacksearch1(__m128i * initOffset, const __m128i *in, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<1)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,13); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,15); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,17); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,19); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,21); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,23); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,25); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,27); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,29); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearch2(__m128i * initOffset, const __m128i *in, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<2)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearch3(__m128i * initOffset, const __m128i *in, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<3)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,15); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,21); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,27); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 3-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,13); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,19); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,25); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 3-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,17); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,23); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearch4(__m128i * initOffset, const __m128i *in, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<4)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearch5(__m128i * initOffset, const __m128i *in, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<5)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,15); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,25); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 5-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,13); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,23); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 5-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,21); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 5-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,19); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 5-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,17); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearch6(__m128i * initOffset, const __m128i *in, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<6)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 6-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 6-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 6-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 6-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearch7(__m128i * initOffset, const __m128i *in, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<7)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,21); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 7-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,17); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 7-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,13); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 7-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,23); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 7-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,19); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 7-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,15); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 7-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearch8(__m128i * initOffset, const __m128i *in, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<8)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearch9(__m128i * initOffset, const __m128i *in, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<9)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 9-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,13); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 9-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,17); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 9-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,21); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 9-7), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 9-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 9-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,15); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 9-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,19); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 9-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearch10(__m128i * initOffset, const __m128i *in, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<10)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 10-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 10-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 10-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 10-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 10-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 10-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 10-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 10-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearch11(__m128i * initOffset, const __m128i *in, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<11)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,13); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,15); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,17); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-7), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,19); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-9), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearch12(__m128i * initOffset, const __m128i *in, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<12)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 12-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 12-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 12-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 12-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 12-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 12-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 12-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 12-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearch13(__m128i * initOffset, const __m128i *in, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<13)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,13); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-7), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,15); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-9), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,17); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-11), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearch14(__m128i * initOffset, const __m128i *in, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<14)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearch15(__m128i * initOffset, const __m128i *in, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<15)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,15); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-13), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,13); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-11), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-9), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-7), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,17); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearch16(__m128i * initOffset, const __m128i *in, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<16)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearch17(__m128i * initOffset, const __m128i *in, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<17)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,17); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-7), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-9), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-11), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-13), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,13); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-15), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,15); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearch18(__m128i * initOffset, const __m128i *in, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<18)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearch19(__m128i * initOffset, const __m128i *in, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<19)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-11), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-17), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,17); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-9), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-15), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,15); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-7), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-13), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,13); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearch20(__m128i * initOffset, const __m128i *in, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<20)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearch21(__m128i * initOffset, const __m128i *in, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<21)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-9), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-19), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-7), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-17), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,17); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-15), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,15); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-13), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,13); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-11), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,11); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearch22(__m128i * initOffset, const __m128i *in, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<22)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearch23(__m128i * initOffset, const __m128i *in, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<23)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-19), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-15), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,15); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-11), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,11); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-7), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-21), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-17), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,17); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-22), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-13), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,13); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-9), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,9); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearch24(__m128i * initOffset, const __m128i *in, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<24)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearch25(__m128i * initOffset, const __m128i *in, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<25)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-11), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,11); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-22), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-15), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,15); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-19), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-23), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-9), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,9); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-13), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,13); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-24), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-17), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,17); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-21), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-7), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,7); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearch26(__m128i * initOffset, const __m128i *in, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<26)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-22), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-24), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-22), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-24), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearch27(__m128i * initOffset, const __m128i *in, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<27)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-22), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-17), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,17); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-7), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,7); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-24), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-19), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-9), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,9); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-26), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-21), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-11), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,11); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-23), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-13), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,13); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-25), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-15), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,15); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,5); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearch28(__m128i * initOffset, const __m128i *in, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<28)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-24), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-24), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-24), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-24), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearch29(__m128i * initOffset, const __m128i *in, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<29)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-26), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-23), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-17), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,17); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-11), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,11); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,5); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-28), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-25), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-22), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-19), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-13), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,13); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-7), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,7); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-27), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-24), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-21), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-15), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,15); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-9), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,9); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,3); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearch30(__m128i * initOffset, const __m128i *in, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<30)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-28), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-26), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-24), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-22), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-28), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-26), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-24), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-22), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearch31(__m128i * initOffset, const __m128i *in, - uint32_t key, uint32_t *presult) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<31)-1); - __m128i conversion = _mm_set1_epi32(2147483648U); - __m128i key4 = _mm_set1_epi32(key - 2147483648U); - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-30), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-29), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-28), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-27), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-26), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-25), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-24), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-23), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-22), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-21), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-19), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-17), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,17); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-15), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,15); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-13), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,13); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-11), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,11); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-9), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,9); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-7), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,7); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,6); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,5); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,4); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,3); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,2); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - tmp = _mm_srli_epi32(InReg,1); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, key, presult); - - *presult = key + 1; - return (128); -} - -static int -iunpacksearch32(__m128i * initOffset, const __m128i *in, - uint32_t key, uint32_t *presult) -{ - uint32_t * in32 = (uint32_t *)in; - int answer = lower_bound(in32, key, 0, 128); - if(in32[answer] < key) { - *presult = key + 1; - return (128); - } - *presult = in32[answer]; - *initOffset = _mm_load_si128(in + 31); - return answer; - -} - - -int -simdsearchd1(__m128i * initOffset, const __m128i *in, uint32_t bit, - uint32_t key, uint32_t *presult) -{ - switch (bit) { - case 0: return iunpacksearch0(initOffset, in, key, presult); - - case 1: return iunpacksearch1(initOffset, in, key, presult); - - case 2: return iunpacksearch2(initOffset, in, key, presult); - - case 3: return iunpacksearch3(initOffset, in, key, presult); - - case 4: return iunpacksearch4(initOffset, in, key, presult); - - case 5: return iunpacksearch5(initOffset, in, key, presult); - - case 6: return iunpacksearch6(initOffset, in, key, presult); - - case 7: return iunpacksearch7(initOffset, in, key, presult); - - case 8: return iunpacksearch8(initOffset, in, key, presult); - - case 9: return iunpacksearch9(initOffset, in, key, presult); - - case 10: return iunpacksearch10(initOffset, in, key, presult); - - case 11: return iunpacksearch11(initOffset, in, key, presult); - - case 12: return iunpacksearch12(initOffset, in, key, presult); - - case 13: return iunpacksearch13(initOffset, in, key, presult); - - case 14: return iunpacksearch14(initOffset, in, key, presult); - - case 15: return iunpacksearch15(initOffset, in, key, presult); - - case 16: return iunpacksearch16(initOffset, in, key, presult); - - case 17: return iunpacksearch17(initOffset, in, key, presult); - - case 18: return iunpacksearch18(initOffset, in, key, presult); - - case 19: return iunpacksearch19(initOffset, in, key, presult); - - case 20: return iunpacksearch20(initOffset, in, key, presult); - - case 21: return iunpacksearch21(initOffset, in, key, presult); - - case 22: return iunpacksearch22(initOffset, in, key, presult); - - case 23: return iunpacksearch23(initOffset, in, key, presult); - - case 24: return iunpacksearch24(initOffset, in, key, presult); - - case 25: return iunpacksearch25(initOffset, in, key, presult); - - case 26: return iunpacksearch26(initOffset, in, key, presult); - - case 27: return iunpacksearch27(initOffset, in, key, presult); - - case 28: return iunpacksearch28(initOffset, in, key, presult); - - case 29: return iunpacksearch29(initOffset, in, key, presult); - - case 30: return iunpacksearch30(initOffset, in, key, presult); - - case 31: return iunpacksearch31(initOffset, in, key, presult); - - case 32: return iunpacksearch32(initOffset, in, key, presult); - - default: break; - } - return (-1); -} - -#endif diff --git a/cpp/simdcomp/src/simdpackedselect.c b/cpp/simdcomp/src/simdpackedselect.c deleted file mode 100644 index 09898f1ff..000000000 --- a/cpp/simdcomp/src/simdpackedselect.c +++ /dev/null @@ -1,15490 +0,0 @@ -/** - * This code is released under a BSD License. - */ -#ifdef __SSE4_1__ -#include "simdintegratedbitpacking.h" -#include - - -SIMDCOMP_ALIGNED(16) int8_t shuffle_mask_bytes[256] = { - 0,1,2,3,0,0,0,0,0,0,0,0,0,0,0,0, - 4,5,6,7,0,0,0,0,0,0,0,0,0,0,0,0, - 8,9,10,11,0,0,0,0,0,0,0,0,0,0,0,0, - 12,13,14,15,0,0,0,0,0,0,0,0,0,0,0,0, - }; - -static const __m128i *shuffle_mask = (__m128i *) shuffle_mask_bytes; - -uint32_t branchlessextract (__m128i out, int i) { - return _mm_cvtsi128_si32(_mm_shuffle_epi8(out,shuffle_mask[i])); -} - -#define PrefixSum(ret, curr, prev) do { \ - const __m128i _tmp1 = _mm_add_epi32(_mm_slli_si128(curr, 8), curr); \ - const __m128i _tmp2 = _mm_add_epi32(_mm_slli_si128(_tmp1, 4), _tmp1); \ - ret = _mm_add_epi32(_tmp2, _mm_shuffle_epi32(prev, 0xff)); \ - } while (0) - -#define CHECK_AND_INCREMENT(i, out, slot) \ - i += 4; \ - if (i > slot) { \ - return branchlessextract (out, slot - (i - 4)); \ - } - - -static uint32_t -iunpackselect1(__m128i * initOffset, const __m128i *in, int slot) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<1)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,13); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,15); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,17); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,19); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,21); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,22); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,23); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,25); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,26); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,27); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,29); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,30); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - - return (0); -} - -static uint32_t -iunpackselect2(__m128i * initOffset, const __m128i *in, int slot) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<2)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,22); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,26); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,22); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,26); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - - return (0); -} - -static uint32_t -iunpackselect3(__m128i * initOffset, const __m128i *in, int slot) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<3)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,15); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,21); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,27); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 3-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,13); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,19); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,22); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,25); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 3-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,17); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,23); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,26); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - - return (0); -} - -static uint32_t -iunpackselect4(__m128i * initOffset, const __m128i *in, int slot) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<4)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - - return (0); -} - -static uint32_t -iunpackselect5(__m128i * initOffset, const __m128i *in, int slot) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<5)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,15); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,25); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 5-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,13); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,23); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 5-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,21); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,26); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 5-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,19); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 5-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,17); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,22); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - - return (0); -} - -static uint32_t -iunpackselect6(__m128i * initOffset, const __m128i *in, int slot) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<6)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 6-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,22); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 6-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 6-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,22); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 6-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - - return (0); -} - -static uint32_t -iunpackselect7(__m128i * initOffset, const __m128i *in, int slot) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<7)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,21); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 7-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,17); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 7-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,13); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 7-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,23); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 7-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,19); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 7-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,15); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,22); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 7-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - - return (0); -} - -static uint32_t -iunpackselect8(__m128i * initOffset, const __m128i *in, int slot) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<8)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - - return (0); -} - -static uint32_t -iunpackselect9(__m128i * initOffset, const __m128i *in, int slot) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<9)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 9-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,13); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,22); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 9-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,17); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 9-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,21); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 9-7), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 9-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 9-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,15); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 9-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,19); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 9-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - - return (0); -} - -static uint32_t -iunpackselect10(__m128i * initOffset, const __m128i *in, int slot) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<10)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 10-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 10-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 10-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 10-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 10-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 10-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 10-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 10-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - - return (0); -} - -static uint32_t -iunpackselect11(__m128i * initOffset, const __m128i *in, int slot) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<11)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,13); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,15); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,17); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-7), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,19); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-9), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - - return (0); -} - -static uint32_t -iunpackselect12(__m128i * initOffset, const __m128i *in, int slot) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<12)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 12-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 12-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 12-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 12-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 12-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 12-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 12-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 12-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - - return (0); -} - -static uint32_t -iunpackselect13(__m128i * initOffset, const __m128i *in, int slot) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<13)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,13); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-7), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,15); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-9), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,17); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-11), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - - return (0); -} - -static uint32_t -iunpackselect14(__m128i * initOffset, const __m128i *in, int slot) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<14)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - - return (0); -} - -static uint32_t -iunpackselect15(__m128i * initOffset, const __m128i *in, int slot) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<15)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,15); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-13), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,13); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-11), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-9), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-7), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,17); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - - return (0); -} - -static uint32_t -iunpackselect16(__m128i * initOffset, const __m128i *in, int slot) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<16)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - - return (0); -} - -static uint32_t -iunpackselect17(__m128i * initOffset, const __m128i *in, int slot) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<17)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,17); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-7), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-9), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-11), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-13), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,13); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-15), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,15); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - - return (0); -} - -static uint32_t -iunpackselect18(__m128i * initOffset, const __m128i *in, int slot) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<18)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - - return (0); -} - -static uint32_t -iunpackselect19(__m128i * initOffset, const __m128i *in, int slot) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<19)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-11), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-17), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,17); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-9), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-15), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,15); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-7), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-13), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,13); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - - return (0); -} - -static uint32_t -iunpackselect20(__m128i * initOffset, const __m128i *in, int slot) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<20)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - - return (0); -} - -static uint32_t -iunpackselect21(__m128i * initOffset, const __m128i *in, int slot) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<21)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-9), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-19), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-7), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-17), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,17); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-15), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,15); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-13), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,13); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-11), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,11); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - - return (0); -} - -static uint32_t -iunpackselect22(__m128i * initOffset, const __m128i *in, int slot) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<22)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - - return (0); -} - -static uint32_t -iunpackselect23(__m128i * initOffset, const __m128i *in, int slot) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<23)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-19), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-15), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,15); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-11), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,11); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-7), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-21), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-17), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,17); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-22), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-13), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,13); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-9), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,9); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - - return (0); -} - -static uint32_t -iunpackselect24(__m128i * initOffset, const __m128i *in, int slot) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<24)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - - return (0); -} - -static uint32_t -iunpackselect25(__m128i * initOffset, const __m128i *in, int slot) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<25)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-11), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,11); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-22), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-15), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,15); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-19), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-23), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-9), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,9); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-13), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,13); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-24), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-17), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,17); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-21), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-7), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,7); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - - return (0); -} - -static uint32_t -iunpackselect26(__m128i * initOffset, const __m128i *in, int slot) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<26)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-22), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-24), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,6); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-22), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-24), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,6); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - - return (0); -} - -static uint32_t -iunpackselect27(__m128i * initOffset, const __m128i *in, int slot) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<27)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-22), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-17), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,17); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-7), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,7); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-24), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-19), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-9), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,9); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-26), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-21), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-11), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,11); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,6); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-23), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-13), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,13); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-25), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-15), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,15); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,5); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - - return (0); -} - -static uint32_t -iunpackselect28(__m128i * initOffset, const __m128i *in, int slot) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<28)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-24), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-24), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-24), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-24), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - - return (0); -} - -static uint32_t -iunpackselect29(__m128i * initOffset, const __m128i *in, int slot) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<29)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-26), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-23), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-17), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,17); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-11), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,11); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,5); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-28), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-25), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-22), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-19), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-13), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,13); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-7), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,7); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-27), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-24), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-21), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-15), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,15); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-9), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,9); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,6); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,3); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - - return (0); -} - -static uint32_t -iunpackselect30(__m128i * initOffset, const __m128i *in, int slot) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<30)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-28), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-26), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-24), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-22), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,6); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,2); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-28), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-26), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-24), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-22), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,6); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,2); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - - return (0); -} - -static uint32_t -iunpackselect31(__m128i * initOffset, const __m128i *in, int slot) -{ - int i = 0; - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<31)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-30), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-29), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-28), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-27), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-26), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-25), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-24), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-23), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-22), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-21), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-19), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-17), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,17); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-15), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,15); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-13), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,13); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-11), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,11); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-9), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,9); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-7), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,7); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,6); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,5); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,4); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,3); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,2); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - tmp = _mm_srli_epi32(InReg,1); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - CHECK_AND_INCREMENT(i, out, slot); - - - return (0); -} - -static uint32_t -iunpackselect32(__m128i * initOffset , const __m128i *in, int slot) -{ - uint32_t *begin = (uint32_t *)in; - *initOffset = _mm_load_si128(in + 31); - return begin[slot]; -} - - -uint32_t -simdselectd1(uint32_t init, const __m128i *in, uint32_t bit, int slot) -{ - __m128i vecinitOffset = _mm_set1_epi32(init); - __m128i * initOffset = &vecinitOffset; - slot &= 127; /* to avoid problems */ - - switch (bit) { - case 0: return _mm_extract_epi32(*initOffset,3); break; - - case 1: return iunpackselect1(initOffset, in, slot); break; - - case 2: return iunpackselect2(initOffset, in, slot); break; - - case 3: return iunpackselect3(initOffset, in, slot); break; - - case 4: return iunpackselect4(initOffset, in, slot); break; - - case 5: return iunpackselect5(initOffset, in, slot); break; - - case 6: return iunpackselect6(initOffset, in, slot); break; - - case 7: return iunpackselect7(initOffset, in, slot); break; - - case 8: return iunpackselect8(initOffset, in, slot); break; - - case 9: return iunpackselect9(initOffset, in, slot); break; - - case 10: return iunpackselect10(initOffset, in, slot); break; - - case 11: return iunpackselect11(initOffset, in, slot); break; - - case 12: return iunpackselect12(initOffset, in, slot); break; - - case 13: return iunpackselect13(initOffset, in, slot); break; - - case 14: return iunpackselect14(initOffset, in, slot); break; - - case 15: return iunpackselect15(initOffset, in, slot); break; - - case 16: return iunpackselect16(initOffset, in, slot); break; - - case 17: return iunpackselect17(initOffset, in, slot); break; - - case 18: return iunpackselect18(initOffset, in, slot); break; - - case 19: return iunpackselect19(initOffset, in, slot); break; - - case 20: return iunpackselect20(initOffset, in, slot); break; - - case 21: return iunpackselect21(initOffset, in, slot); break; - - case 22: return iunpackselect22(initOffset, in, slot); break; - - case 23: return iunpackselect23(initOffset, in, slot); break; - - case 24: return iunpackselect24(initOffset, in, slot); break; - - case 25: return iunpackselect25(initOffset, in, slot); break; - - case 26: return iunpackselect26(initOffset, in, slot); break; - - case 27: return iunpackselect27(initOffset, in, slot); break; - - case 28: return iunpackselect28(initOffset, in, slot); break; - - case 29: return iunpackselect29(initOffset, in, slot); break; - - case 30: return iunpackselect30(initOffset, in, slot); break; - - case 31: return iunpackselect31(initOffset, in, slot); break; - - case 32: return iunpackselect32(initOffset, in, slot); break; - - default: break; - } - - return (-1); -} - -static void -iunpackscan1(__m128i * initOffset, const __m128i *in) -{ - - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<1)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,13); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,15); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,17); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,19); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,21); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,22); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,23); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,25); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,26); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,27); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,29); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,30); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - -} - -static void -iunpackscan2(__m128i * initOffset, const __m128i *in) -{ - - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<2)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,22); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,26); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,22); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,26); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - -} - -static void -iunpackscan3(__m128i * initOffset, const __m128i *in) -{ - - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<3)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,15); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,21); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,27); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 3-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,13); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,19); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,22); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,25); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 3-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,17); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,23); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,26); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - -} - -static void -iunpackscan4(__m128i * initOffset, const __m128i *in) -{ - - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<4)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - -} - -static void -iunpackscan5(__m128i * initOffset, const __m128i *in) -{ - - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<5)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,15); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,25); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 5-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,13); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,23); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 5-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,21); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,26); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 5-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,19); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 5-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,17); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,22); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - -} - -static void -iunpackscan6(__m128i * initOffset, const __m128i *in) -{ - - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<6)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 6-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,22); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 6-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 6-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,22); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 6-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - -} - -static void -iunpackscan7(__m128i * initOffset, const __m128i *in) -{ - - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<7)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,21); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 7-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,17); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 7-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,13); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 7-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,23); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 7-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,19); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 7-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,15); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,22); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 7-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - -} - -static void -iunpackscan8(__m128i * initOffset, const __m128i *in) -{ - - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<8)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - -} - -static void -iunpackscan9(__m128i * initOffset, const __m128i *in) -{ - - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<9)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 9-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,13); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,22); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 9-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,17); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 9-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,21); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 9-7), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 9-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 9-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,15); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 9-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,19); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 9-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - -} - -static void -iunpackscan10(__m128i * initOffset, const __m128i *in) -{ - - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<10)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 10-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 10-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 10-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 10-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 10-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 10-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 10-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 10-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - -} - -static void -iunpackscan11(__m128i * initOffset, const __m128i *in) -{ - - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<11)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,13); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,15); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,17); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-7), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,19); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-9), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 11-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - -} - -static void -iunpackscan12(__m128i * initOffset, const __m128i *in) -{ - - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<12)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 12-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 12-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 12-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 12-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 12-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 12-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 12-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 12-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - -} - -static void -iunpackscan13(__m128i * initOffset, const __m128i *in) -{ - - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<13)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,13); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-7), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,15); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-9), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,17); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-11), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,18); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 13-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - -} - -static void -iunpackscan14(__m128i * initOffset, const __m128i *in) -{ - - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<14)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 14-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - -} - -static void -iunpackscan15(__m128i * initOffset, const __m128i *in) -{ - - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<15)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,15); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-13), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,13); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-11), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-9), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-7), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 15-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,17); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - -} - -static void -iunpackscan16(__m128i * initOffset, const __m128i *in) -{ - - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<16)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - -} - -static void -iunpackscan17(__m128i * initOffset, const __m128i *in) -{ - - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<17)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,17); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,14); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-7), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-9), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-11), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-13), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,13); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 17-15), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,15); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - -} - -static void -iunpackscan18(__m128i * initOffset, const __m128i *in) -{ - - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<18)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 18-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - -} - -static void -iunpackscan19(__m128i * initOffset, const __m128i *in) -{ - - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<19)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-11), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,11); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-17), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,17); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-9), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-15), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,15); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-7), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 19-13), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,13); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - -} - -static void -iunpackscan20(__m128i * initOffset, const __m128i *in) -{ - - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<20)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 20-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - - -} - -static void -iunpackscan21(__m128i * initOffset, const __m128i *in) -{ - - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<21)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,10); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-9), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,9); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-19), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-7), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-17), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,17); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-15), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,15); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-13), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,13); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 21-11), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,11); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - - -} - -static void -iunpackscan22(__m128i * initOffset, const __m128i *in) -{ - - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<22)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 22-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - - -} - -static void -iunpackscan23(__m128i * initOffset, const __m128i *in) -{ - - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<23)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-19), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-15), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,15); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-11), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,11); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-7), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,7); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-21), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-17), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,17); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-22), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-13), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,13); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 23-9), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,9); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - - -} - -static void -iunpackscan24(__m128i * initOffset, const __m128i *in) -{ - - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<24)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 24-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - - -} - -static void -iunpackscan25(__m128i * initOffset, const __m128i *in) -{ - - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<25)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-11), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,11); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-22), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-15), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,15); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-19), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,5); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-23), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-9), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,9); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-13), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,13); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,6); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-24), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-17), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,17); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-21), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 25-7), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,7); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - - -} - -static void -iunpackscan26(__m128i * initOffset, const __m128i *in) -{ - - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<26)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-22), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-24), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,6); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-22), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-24), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 26-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,6); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - - -} - -static void -iunpackscan27(__m128i * initOffset, const __m128i *in) -{ - - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<27)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-22), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-17), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,17); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-7), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,7); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-24), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-19), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-9), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,9); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-26), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-21), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-11), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,11); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,6); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-23), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-13), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,13); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,3); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-25), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-15), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,15); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 27-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,5); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - - -} - -static void -iunpackscan28(__m128i * initOffset, const __m128i *in) -{ - - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<28)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-24), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-24), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-24), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-24), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 28-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - - -} - -static void -iunpackscan29(__m128i * initOffset, const __m128i *in) -{ - - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<29)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-26), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-23), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-17), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,17); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-11), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,11); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,5); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,2); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-28), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-25), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-22), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-19), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-13), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,13); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-7), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,7); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,1); - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-27), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-24), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-21), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-15), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,15); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-9), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,9); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,6); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 29-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,3); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - - -} - -static void -iunpackscan30(__m128i * initOffset, const __m128i *in) -{ - - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<30)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-28), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-26), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-24), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-22), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,6); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,2); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-28), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-26), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-24), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-22), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,6); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 30-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,2); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - - -} - -static void -iunpackscan31(__m128i * initOffset, const __m128i *in) -{ - - __m128i InReg = _mm_loadu_si128(in); - __m128i out; - __m128i tmp; - __m128i mask = _mm_set1_epi32((1U<<31)-1); - - tmp = InReg; - out = _mm_and_si128(tmp, mask); - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,31); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-30), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,30); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-29), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,29); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-28), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,28); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-27), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,27); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-26), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,26); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-25), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,25); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-24), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,24); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-23), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,23); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-22), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,22); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-21), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,21); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-20), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,20); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-19), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,19); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-18), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,18); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-17), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,17); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-16), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,16); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-15), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,15); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-14), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,14); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-13), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,13); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-12), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,12); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-11), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,11); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-10), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,10); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-9), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,9); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-8), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,8); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-7), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,7); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-6), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,6); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-5), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,5); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-4), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,4); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-3), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,3); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-2), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,2); - out = tmp; - ++in; InReg = _mm_loadu_si128(in); - out = _mm_or_si128(out, _mm_and_si128(_mm_slli_epi32(InReg, 31-1), mask)); - - PrefixSum(out, out, *initOffset); - *initOffset = out; - - - tmp = _mm_srli_epi32(InReg,1); - out = tmp; - PrefixSum(out, out, *initOffset); - *initOffset = out; - - -} - -static void -iunpackscan32(__m128i * initOffset , const __m128i *in) -{ - *initOffset = _mm_load_si128(in + 31); -} - - -void -simdscand1(__m128i * initOffset, const __m128i *in, uint32_t bit) -{ - switch (bit) { - case 0: return; break; - - case 1: iunpackscan1(initOffset, in); break; - - case 2: iunpackscan2(initOffset, in); break; - - case 3: iunpackscan3(initOffset, in); break; - - case 4: iunpackscan4(initOffset, in); break; - - case 5: iunpackscan5(initOffset, in); break; - - case 6: iunpackscan6(initOffset, in); break; - - case 7: iunpackscan7(initOffset, in); break; - - case 8: iunpackscan8(initOffset, in); break; - - case 9: iunpackscan9(initOffset, in); break; - - case 10: iunpackscan10(initOffset, in); break; - - case 11: iunpackscan11(initOffset, in); break; - - case 12: iunpackscan12(initOffset, in); break; - - case 13: iunpackscan13(initOffset, in); break; - - case 14: iunpackscan14(initOffset, in); break; - - case 15: iunpackscan15(initOffset, in); break; - - case 16: iunpackscan16(initOffset, in); break; - - case 17: iunpackscan17(initOffset, in); break; - - case 18: iunpackscan18(initOffset, in); break; - - case 19: iunpackscan19(initOffset, in); break; - - case 20: iunpackscan20(initOffset, in); break; - - case 21: iunpackscan21(initOffset, in); break; - - case 22: iunpackscan22(initOffset, in); break; - - case 23: iunpackscan23(initOffset, in); break; - - case 24: iunpackscan24(initOffset, in); break; - - case 25: iunpackscan25(initOffset, in); break; - - case 26: iunpackscan26(initOffset, in); break; - - case 27: iunpackscan27(initOffset, in); break; - - case 28: iunpackscan28(initOffset, in); break; - - case 29: iunpackscan29(initOffset, in); break; - - case 30: iunpackscan30(initOffset, in); break; - - case 31: iunpackscan31(initOffset, in); break; - - case 32: iunpackscan32(initOffset, in); break; - - default: break; - } - - return ; -} - -#endif diff --git a/cpp/simdcomp/tests/unit.c b/cpp/simdcomp/tests/unit.c deleted file mode 100644 index 341ef4e37..000000000 --- a/cpp/simdcomp/tests/unit.c +++ /dev/null @@ -1,900 +0,0 @@ -/** - * This code is released under a BSD License. - */ -#include -#include -#include -#include "simdcomp.h" - - - -int testshortpack() { - int bit; - size_t i; - size_t length; - __m128i * bb; - srand(0); - printf("testshortpack\n"); - for (bit = 0; bit < 32; ++bit) { - const size_t N = 128; - uint32_t * data = malloc(N * sizeof(uint32_t)); - uint32_t * backdata = malloc(N * sizeof(uint32_t)); - uint32_t * buffer = malloc((2 * N + 1024) * sizeof(uint32_t)); - - for (i = 0; i < N; ++i) { - data[i] = rand() & ((1 << bit) - 1); - } - for (length = 0; length <= N; ++length) { - for (i = 0; i < N; ++i) { - backdata[i] = 0; - } - bb = simdpack_shortlength(data, length, (__m128i *) buffer, - bit); - if((bb - (__m128i *) buffer) * sizeof(__m128i) != (unsigned) simdpack_compressedbytes(length,bit)) { - printf("bug\n"); - return -1; - } - simdunpack_shortlength((__m128i *) buffer, length, - backdata, bit); - for (i = 0; i < length; ++i) { - - if (data[i] != backdata[i]) { - printf("bug\n"); - return -1; - } - } - } - free(data); - free(backdata); - free(buffer); - } - return 0; -} - -int testlongpack() { - int bit; - size_t i; - size_t length; - __m128i * bb; - srand(0); - printf("testlongpack\n"); - for (bit = 0; bit < 32; ++bit) { - const size_t N = 2048; - uint32_t * data = malloc(N * sizeof(uint32_t)); - uint32_t * backdata = malloc(N * sizeof(uint32_t)); - uint32_t * buffer = malloc((2 * N + 1024) * sizeof(uint32_t)); - - for (i = 0; i < N; ++i) { - data[i] = rand() & ((1 << bit) - 1); - } - for (length = 0; length <= N; ++length) { - for (i = 0; i < N; ++i) { - backdata[i] = 0; - } - bb = simdpack_length(data, length, (__m128i *) buffer, - bit); - if((bb - (__m128i *) buffer) * sizeof(__m128i) != (unsigned) simdpack_compressedbytes(length,bit)) { - printf("bug\n"); - return -1; - } - simdunpack_length((__m128i *) buffer, length, - backdata, bit); - for (i = 0; i < length; ++i) { - - if (data[i] != backdata[i]) { - printf("bug\n"); - return -1; - } - } - } - free(data); - free(backdata); - free(buffer); - } - return 0; -} - - - -int testset() { - int bit; - size_t i; - const size_t N = 128; - uint32_t * data = malloc(N * sizeof(uint32_t)); - uint32_t * backdata = malloc(N * sizeof(uint32_t)); - uint32_t * buffer = malloc((2 * N + 1024) * sizeof(uint32_t)); - - srand(0); - - for (bit = 0; bit < 32; ++bit) { - printf("simple set %d \n",bit); - - for (i = 0; i < N; ++i) { - data[i] = rand() & ((1 << bit) - 1); - } - for (i = 0; i < N; ++i) { - backdata[i] = 0; - } - simdpack(data, (__m128i *) buffer, bit); - simdunpack((__m128i *) buffer, backdata, bit); - for (i = 0; i < N; ++i) { - if (data[i] != backdata[i]) { - printf("bug\n"); - return -1; - } - } - - for(i = N ; i > 0; i--) { - simdfastset((__m128i *) buffer, bit, data[N - i], i - 1); - } - simdunpack((__m128i *) buffer, backdata, bit); - for (i = 0; i < N; ++i) { - if (data[i] != backdata[N - i - 1]) { - printf("bug\n"); - return -1; - } - } - simdpack(data, (__m128i *) buffer, bit); - for(i = 1 ; i <= N; i++) { - simdfastset((__m128i *) buffer, bit, data[i - 1], i - 1); - } - simdunpack((__m128i *) buffer, backdata, bit); - for (i = 0; i < N; ++i) { - if (data[i] != backdata[i]) { - printf("bug\n"); - return -1; - } - } - - } - free(data); - free(backdata); - free(buffer); - - return 0; -} - -#ifdef __SSE4_1__ - -int testsetd1() { - int bit; - size_t i; - uint32_t newvalue; - const size_t N = 128; - uint32_t * data = malloc(N * sizeof(uint32_t)); - uint32_t * datazeroes = malloc(N * sizeof(uint32_t)); - - uint32_t * backdata = malloc(N * sizeof(uint32_t)); - uint32_t * buffer = malloc((2 * N + 1024) * sizeof(uint32_t)); - - srand(0); - for (bit = 0; bit < 32; ++bit) { - printf("simple set d1 %d \n",bit); - data[0] = rand() & ((1 << bit) - 1); - datazeroes[0] = 0; - - for (i = 1; i < N; ++i) { - data[i] = data[i - 1] + (rand() & ((1 << bit) - 1)); - datazeroes[i] = 0; - } - for (i = 0; i < N; ++i) { - backdata[i] = 0; - } - simdpackd1(0,datazeroes, (__m128i *) buffer, bit); - for(i = 1 ; i <= N; i++) { - simdfastsetd1(0,(__m128i *) buffer, bit, data[i - 1], i - 1); - newvalue = simdselectd1(0, (const __m128i *) buffer, bit,i - 1); - if( newvalue != data[i-1] ) { - printf("bad set-select\n"); - return -1; - } - } - simdunpackd1(0,(__m128i *) buffer, backdata, bit); - for (i = 0; i < N; ++i) { - if (data[i] != backdata[i]) - return -1; - } - } - free(data); - free(backdata); - free(buffer); - free(datazeroes); - return 0; -} -#endif - -int testsetFOR() { - int bit; - size_t i; - uint32_t newvalue; - const size_t N = 128; - uint32_t * data = malloc(N * sizeof(uint32_t)); - uint32_t * datazeroes = malloc(N * sizeof(uint32_t)); - - uint32_t * backdata = malloc(N * sizeof(uint32_t)); - uint32_t * buffer = malloc((2 * N + 1024) * sizeof(uint32_t)); - - srand(0); - for (bit = 0; bit < 32; ++bit) { - printf("simple set FOR %d \n",bit); - for (i = 0; i < N; ++i) { - data[i] = (rand() & ((1 << bit) - 1)); - datazeroes[i] = 0; - } - for (i = 0; i < N; ++i) { - backdata[i] = 0; - } - simdpackFOR(0,datazeroes, (__m128i *) buffer, bit); - for(i = 1 ; i <= N; i++) { - simdfastsetFOR(0,(__m128i *) buffer, bit, data[i - 1], i - 1); - newvalue = simdselectFOR(0, (const __m128i *) buffer, bit,i - 1); - if( newvalue != data[i-1] ) { - printf("bad set-select\n"); - return -1; - } - } - simdunpackFOR(0,(__m128i *) buffer, backdata, bit); - for (i = 0; i < N; ++i) { - if (data[i] != backdata[i]) - return -1; - } - } - free(data); - free(backdata); - free(buffer); - free(datazeroes); - return 0; -} - -int testshortFORpack() { - int bit; - size_t i; - __m128i * rb; - size_t length; - uint32_t offset = 7; - srand(0); - for (bit = 0; bit < 32; ++bit) { - const size_t N = 128; - uint32_t * data = malloc(N * sizeof(uint32_t)); - uint32_t * backdata = malloc(N * sizeof(uint32_t)); - uint32_t * buffer = malloc((2 * N + 1024) * sizeof(uint32_t)); - - for (i = 0; i < N; ++i) { - data[i] = (rand() & ((1 << bit) - 1)) + offset; - } - for (length = 0; length <= N; ++length) { - for (i = 0; i < N; ++i) { - backdata[i] = 0; - } - rb = simdpackFOR_length(offset,data, length, (__m128i *) buffer, - bit); - if(((rb - (__m128i *) buffer)*sizeof(__m128i)) != (unsigned) simdpackFOR_compressedbytes(length,bit)) { - return -1; - } - simdunpackFOR_length(offset,(__m128i *) buffer, length, - backdata, bit); - for (i = 0; i < length; ++i) { - - if (data[i] != backdata[i]) - return -1; - } - } - free(data); - free(backdata); - free(buffer); - } - return 0; -} - - -#ifdef __AVX2__ - -int testbabyavx() { - int bit; - int trial; - unsigned int i,j; - const size_t N = AVXBlockSize; - srand(0); - printf("testbabyavx\n"); - printf("bit = "); - for (bit = 0; bit < 32; ++bit) { - printf(" %d ",bit); - fflush(stdout); - for(trial = 0; trial < 100; ++trial) { - uint32_t * data = malloc(N * sizeof(uint32_t)+ 64 * sizeof(uint32_t)); - uint32_t * backdata = malloc(N * sizeof(uint32_t) + 64 * sizeof(uint32_t) ); - __m256i * buffer = malloc((2 * N + 1024) * sizeof(uint32_t) + 32); - - for (i = 0; i < N; ++i) { - data[i] = rand() & ((uint32_t)(1 << bit) - 1); - } - for (i = 0; i < N; ++i) { - backdata[i] = 0; - } - if(avxmaxbits(data) != maxbits_length(data,N)) { - printf("avxmaxbits is buggy\n"); - return -1; - } - - avxpackwithoutmask(data, buffer, bit); - avxunpack(buffer, backdata, bit); - for (i = 0; i < AVXBlockSize; ++i) { - if (data[i] != backdata[i]) { - printf("bug\n"); - for (j = 0; j < N; ++j) { - if (data[j] != backdata[j]) { - printf("data[%d]=%d v.s. backdata[%d]=%d\n",j,data[j],j,backdata[j]); - } else { - printf("data[%d]=%d\n",j,data[j]); - } - } - return -1; - } - } - free(data); - free(backdata); - free(buffer); - } - } - printf("\n"); - return 0; -} - -int testavx2() { - int N = 5000 * AVXBlockSize, gap; - __m256i * buffer = malloc(AVXBlockSize * sizeof(uint32_t)); - uint32_t * datain = malloc(N * sizeof(uint32_t)); - uint32_t * backbuffer = malloc(AVXBlockSize * sizeof(uint32_t)); - for (gap = 1; gap <= 387420489; gap *= 3) { - int k; - printf(" gap = %u \n", gap); - for (k = 0; k < N; ++k) - datain[k] = k * gap; - for (k = 0; k * AVXBlockSize < N; ++k) { - /* - First part works for general arrays (sorted or unsorted) - */ - int j; - /* we compute the bit width */ - const uint32_t b = avxmaxbits(datain + k * AVXBlockSize); - if(avxmaxbits(datain + k * AVXBlockSize) != maxbits_length(datain + k * AVXBlockSize,AVXBlockSize)) { - printf("avxmaxbits is buggy %d %d \n", - avxmaxbits(datain + k * AVXBlockSize), - maxbits_length(datain + k * AVXBlockSize,AVXBlockSize)); - return -1; - } - printf("bit width = %d\n",b); - - - /* we read 256 integers at "datain + k * AVXBlockSize" and - write b 256-bit vectors at "buffer" */ - avxpackwithoutmask(datain + k * AVXBlockSize, buffer, b); - /* we read back b1 128-bit vectors at "buffer" and write 128 integers at backbuffer */ - avxunpack(buffer, backbuffer, b);/* uncompressed */ - for (j = 0; j < AVXBlockSize; ++j) { - if (backbuffer[j] != datain[k * AVXBlockSize + j]) { - int i; - printf("bug in avxpack\n"); - for(i = 0; i < AVXBlockSize; ++i) { - printf("data[%d]=%d got back %d %s\n",i, - datain[k * AVXBlockSize + i],backbuffer[i], - datain[k * AVXBlockSize + i]!=backbuffer[i]?"bug":""); - } - return -2; - } - } - } - } - free(buffer); - free(datain); - free(backbuffer); - printf("Code looks good.\n"); - return 0; -} -#endif /* avx2 */ - -int test() { - int N = 5000 * SIMDBlockSize, gap; - __m128i * buffer = malloc(SIMDBlockSize * sizeof(uint32_t)); - uint32_t * datain = malloc(N * sizeof(uint32_t)); - uint32_t * backbuffer = malloc(SIMDBlockSize * sizeof(uint32_t)); - for (gap = 1; gap <= 387420489; gap *= 3) { - int k; - printf(" gap = %u \n", gap); - for (k = 0; k < N; ++k) - datain[k] = k * gap; - for (k = 0; k * SIMDBlockSize < N; ++k) { - /* - First part works for general arrays (sorted or unsorted) - */ - int j; - /* we compute the bit width */ - const uint32_t b = maxbits(datain + k * SIMDBlockSize); - /* we read 128 integers at "datain + k * SIMDBlockSize" and - write b 128-bit vectors at "buffer" */ - simdpackwithoutmask(datain + k * SIMDBlockSize, buffer, b); - /* we read back b1 128-bit vectors at "buffer" and write 128 integers at backbuffer */ - simdunpack(buffer, backbuffer, b);/* uncompressed */ - for (j = 0; j < SIMDBlockSize; ++j) { - if (backbuffer[j] != datain[k * SIMDBlockSize + j]) { - printf("bug in simdpack\n"); - return -2; - } - } - - { - /* - next part assumes that the data is sorted (uses differential coding) - */ - uint32_t offset = 0; - /* we compute the bit width */ - const uint32_t b1 = simdmaxbitsd1(offset, - datain + k * SIMDBlockSize); - /* we read 128 integers at "datain + k * SIMDBlockSize" and - write b1 128-bit vectors at "buffer" */ - simdpackwithoutmaskd1(offset, datain + k * SIMDBlockSize, buffer, - b1); - /* we read back b1 128-bit vectors at "buffer" and write 128 integers at backbuffer */ - simdunpackd1(offset, buffer, backbuffer, b1); - for (j = 0; j < SIMDBlockSize; ++j) { - if (backbuffer[j] != datain[k * SIMDBlockSize + j]) { - printf("bug in simdpack d1\n"); - return -3; - } - } - offset = datain[k * SIMDBlockSize + SIMDBlockSize - 1]; - } - } - } - free(buffer); - free(datain); - free(backbuffer); - printf("Code looks good.\n"); - return 0; -} - -#ifdef __SSE4_1__ -int testFOR() { - int N = 5000 * SIMDBlockSize, gap; - __m128i * buffer = malloc(SIMDBlockSize * sizeof(uint32_t)); - uint32_t * datain = malloc(N * sizeof(uint32_t)); - uint32_t * backbuffer = malloc(SIMDBlockSize * sizeof(uint32_t)); - uint32_t tmax, tmin, tb; - for (gap = 1; gap <= 387420489; gap *= 2) { - int k; - printf(" gap = %u \n", gap); - for (k = 0; k < N; ++k) - datain[k] = k * gap; - for (k = 0; k * SIMDBlockSize < N; ++k) { - int j; - simdmaxmin_length(datain + k * SIMDBlockSize,SIMDBlockSize,&tmin,&tmax); - /* we compute the bit width */ - tb = bits(tmax - tmin); - - - /* we read 128 integers at "datain + k * SIMDBlockSize" and - write b 128-bit vectors at "buffer" */ - simdpackFOR(tmin,datain + k * SIMDBlockSize, buffer, tb); - - for (j = 0; j < SIMDBlockSize; ++j) { - uint32_t selectedvalue = simdselectFOR(tmin,buffer,tb,j); - if (selectedvalue != datain[k * SIMDBlockSize + j]) { - printf("bug in simdselectFOR\n"); - return -3; - } - } - /* we read back b1 128-bit vectors at "buffer" and write 128 integers at backbuffer */ - simdunpackFOR(tmin,buffer, backbuffer, tb);/* uncompressed */ - for (j = 0; j < SIMDBlockSize; ++j) { - if (backbuffer[j] != datain[k * SIMDBlockSize + j]) { - printf("bug in simdpackFOR\n"); - return -2; - } - } - } - } - free(buffer); - free(datain); - free(backbuffer); - printf("Code looks good.\n"); - return 0; -} -#endif - -#define MAX 300 -int test_simdmaxbitsd1_length() { - uint32_t result, buffer[MAX + 1]; - int i, j; - - memset(&buffer[0], 0xff, sizeof(buffer)); - - /* this test creates buffers of different length; each buffer is - * initialized to result in the following deltas: - * length 1: 2 - * length 2: 1 2 - * length 3: 1 1 2 - * length 4: 1 1 1 2 - * length 5: 1 1 1 1 2 - * etc. Each sequence's "maxbits" is 2. */ - for (i = 0; i < MAX; i++) { - for (j = 0; j < i; j++) - buffer[j] = j + 1; - buffer[i] = i + 2; - - result = simdmaxbitsd1_length(0, &buffer[0], i + 1); - if (result != 2) { - printf("simdmaxbitsd1_length: unexpected result %u in loop %d\n", - result, i); - return -1; - } - } - printf("simdmaxbitsd1_length: ok\n"); - return 0; -} - -int uint32_cmp(const void *a, const void *b) -{ - const uint32_t *ia = (const uint32_t *)a; - const uint32_t *ib = (const uint32_t *)b; - if(*ia < *ib) - return -1; - else if (*ia > *ib) - return 1; - return 0; -} - -#ifdef __SSE4_1__ -int test_simdpackedsearch() { - uint32_t buffer[128]; - uint32_t result = 0; - int b, i; - uint32_t init = 0; - __m128i initial = _mm_set1_epi32(init); - - /* initialize the buffer */ - for (i = 0; i < 128; i++) - buffer[i] = (uint32_t)(i + 1); - - /* this test creates delta encoded buffers with different bits, then - * performs lower bound searches for each key */ - for (b = 1; b <= 32; b++) { - uint32_t out[128]; - /* delta-encode to 'i' bits */ - simdpackwithoutmaskd1(init, buffer, (__m128i *)out, b); - initial = _mm_setzero_si128(); - printf("simdsearchd1: %d bits\n", b); - - /* now perform the searches */ - initial = _mm_set1_epi32(init); - assert(simdsearchd1(&initial, (__m128i *)out, b, 0, &result) == 0); - assert(result > 0); - - for (i = 1; i <= 128; i++) { - initial = _mm_set1_epi32(init); - assert(simdsearchd1(&initial, (__m128i *)out, b, - (uint32_t)i, &result) == i - 1); - assert(result == (unsigned)i); - } - initial = _mm_set1_epi32(init); - assert(simdsearchd1(&initial, (__m128i *)out, b, 200, &result) - == 128); - assert(result > 200); - } - printf("simdsearchd1: ok\n"); - return 0; -} - -int test_simdpackedsearchFOR() { - uint32_t buffer[128]; - uint32_t result = 0; - int b; - uint32_t i; - uint32_t maxv, tmin, tmax, tb; - uint32_t out[128]; - - /* this test creates delta encoded buffers with different bits, then - * performs lower bound searches for each key */ - for (b = 1; b <= 32; b++) { - /* initialize the buffer */ - maxv = (b == 32) - ? 0xFFFFFFFF - : ((1U< 0) - assert(buffer[pos - 1] < buffer[i]); - assert(result == buffer[i]); - } - for (i = 0; i < 128; i++) { - int pos; - if(buffer[i] == 0) continue; - initial = _mm_set1_epi32(init); - pos = simdsearchd1(&initial, (__m128i *)out, b, - buffer[i] - 1, &result); - assert(pos == simdsearchwithlengthd1(init, (__m128i *)out, b, 128, - buffer[i] - 1, &result)); - assert(buffer[pos] >= buffer[i] - 1); - if(pos > 0) - assert(buffer[pos - 1] < buffer[i] - 1); - assert(result == buffer[pos]); - } - for (i = 0; i < 128; i++) { - int pos; - if (buffer[i] + 1 == 0) - continue; - initial = _mm_set1_epi32(init); - pos = simdsearchd1(&initial, (__m128i *) out, b, - buffer[i] + 1, &result); - assert(pos == simdsearchwithlengthd1(init, (__m128i *)out, b, 128, - buffer[i] + 1, &result)); - if(pos == 128) { - assert(buffer[i] == buffer[127]); - } else { - assert(buffer[pos] >= buffer[i] + 1); - if (pos > 0) - assert(buffer[pos - 1] < buffer[i] + 1); - assert(result == buffer[pos]); - } - } - } - printf("advanced simdsearchd1: ok\n"); - return 0; -} - -int test_simdpackedselect() { - uint32_t buffer[128]; - uint32_t initial = 33; - int b, i; - - /* initialize the buffer */ - for (i = 0; i < 128; i++) - buffer[i] = (uint32_t)(initial + i); - - /* this test creates delta encoded buffers with different bits, then - * performs lower bound searches for each key */ - for (b = 1; b <= 32; b++) { - uint32_t out[128]; - /* delta-encode to 'i' bits */ - simdpackwithoutmaskd1(initial, buffer, (__m128i *)out, b); - - printf("simdselectd1: %d bits\n", b); - - /* now perform the searches */ - for (i = 0; i < 128; i++) { - assert(simdselectd1(initial, (__m128i *)out, b, (uint32_t)i) - == initial + i); - } - } - printf("simdselectd1: ok\n"); - return 0; -} - -int test_simdpackedselect_advanced() { - uint32_t buffer[128]; - uint32_t initial = 33; - uint32_t b; - int i; - - /* this test creates delta encoded buffers with different bits, then - * performs lower bound searches for each key */ - for (b = 0; b <= 32; b++) { - uint32_t prev = initial; - uint32_t out[128]; - /* initialize the buffer */ - for (i = 0; i < 128; i++) { - buffer[i] = ((uint32_t)(165576 * i)) ; - if(b < 32) buffer[i] %= (1< -#include -#include -#include "simdcomp.h" - - -#define get_random_char() (uint8_t)(rand() % 256); - - -int main() { - int N = 5000 * SIMDBlockSize, gap; - __m128i * buffer = malloc(SIMDBlockSize * sizeof(uint32_t)); - uint32_t * datain = malloc(N * sizeof(uint32_t)); - uint32_t * backbuffer = malloc(SIMDBlockSize * sizeof(uint32_t)); - - srand(time(NULL)); - - for (gap = 1; gap <= 387420489; gap *= 3) { - int k; - printf(" gap = %u \n", gap); - - /* simulate some random character string, don't care about endiannes */ - for (k = 0; k < N; ++k) { - uint8_t _tmp[4]; - - _tmp[0] = get_random_char(); - _tmp[1] = get_random_char(); - _tmp[2] = get_random_char(); - _tmp[3] = get_random_char(); - - memmove(&datain[k], _tmp, 4); - } - for (k = 0; k * SIMDBlockSize < N; ++k) { - /* - First part works for general arrays (sorted or unsorted) - */ - int j; - /* we compute the bit width */ - const uint32_t b = maxbits(datain + k * SIMDBlockSize); - /* we read 128 integers at "datain + k * SIMDBlockSize" and - write b 128-bit vectors at "buffer" */ - simdpackwithoutmask(datain + k * SIMDBlockSize, buffer, b); - /* we read back b1 128-bit vectors at "buffer" and write 128 integers at backbuffer */ - simdunpack(buffer, backbuffer, b);/* uncompressed */ - for (j = 0; j < SIMDBlockSize; ++j) { - uint8_t chars_back[4]; - uint8_t chars_in[4]; - - memmove(chars_back, &backbuffer[j], 4); - memmove(chars_in, &datain[k * SIMDBlockSize + j], 4); - - if (chars_in[0] != chars_back[0] - || chars_in[1] != chars_back[1] - || chars_in[2] != chars_back[2] - || chars_in[3] != chars_back[3]) { - printf("bug in simdpack\n"); - return -2; - } - } - - { - /* - next part assumes that the data is sorted (uses differential coding) - */ - uint32_t offset = 0; - /* we compute the bit width */ - const uint32_t b1 = simdmaxbitsd1(offset, - datain + k * SIMDBlockSize); - /* we read 128 integers at "datain + k * SIMDBlockSize" and - write b1 128-bit vectors at "buffer" */ - simdpackwithoutmaskd1(offset, datain + k * SIMDBlockSize, buffer, - b1); - /* we read back b1 128-bit vectors at "buffer" and write 128 integers at backbuffer */ - simdunpackd1(offset, buffer, backbuffer, b1); - for (j = 0; j < SIMDBlockSize; ++j) { - uint8_t chars_back[4]; - uint8_t chars_in[4]; - - memmove(chars_back, &backbuffer[j], 4); - memmove(chars_in, &datain[k * SIMDBlockSize + j], 4); - - if (chars_in[0] != chars_back[0] - || chars_in[1] != chars_back[1] - || chars_in[2] != chars_back[2] - || chars_in[3] != chars_back[3]) { - printf("bug in simdpack\n"); - return -3; - } - } - offset = datain[k * SIMDBlockSize + SIMDBlockSize - 1]; - } - } - } - free(buffer); - free(datain); - free(backbuffer); - printf("Code looks good.\n"); - return 0; -} diff --git a/cpp/simdcomp_wrapper.c b/cpp/simdcomp_wrapper.c deleted file mode 100644 index 4530e3f3b..000000000 --- a/cpp/simdcomp_wrapper.c +++ /dev/null @@ -1,42 +0,0 @@ -#include "simdcomp.h" -#include "simdcomputil.h" - -// assumes datain has a size of 128 uint32 -// and that buffer is large enough to host the data. -size_t compress_sorted( - const uint32_t* datain, - uint8_t* output, - const uint32_t offset) { - const uint32_t b = simdmaxbitsd1(offset, datain); - *output++ = b; - simdpackwithoutmaskd1(offset, datain, (__m128i *) output, b); - return 1 + b * sizeof(__m128i); -} - -// assumes datain has a size of 128 uint32 -// and that buffer is large enough to host the data. -size_t uncompress_sorted( - const uint8_t* compressed_data, - uint32_t* output, - uint32_t offset) { - const uint32_t b = *compressed_data++; - simdunpackd1(offset, (__m128i *)compressed_data, output, b); - return 1 + b * sizeof(__m128i); -} - -size_t compress_unsorted( - const uint32_t* datain, - uint8_t* output) { - const uint32_t b = maxbits(datain); - *output++ = b; - simdpackwithoutmask(datain, (__m128i *) output, b); - return 1 + b * sizeof(__m128i); -} - -size_t uncompress_unsorted( - const uint8_t* compressed_data, - uint32_t* output) { - const uint32_t b = *compressed_data++; - simdunpack((__m128i *)compressed_data, output, b); - return 1 + b * sizeof(__m128i); -} diff --git a/cpp/streamvbyte/.gitignore b/cpp/streamvbyte/.gitignore deleted file mode 100644 index bbf313b25..000000000 --- a/cpp/streamvbyte/.gitignore +++ /dev/null @@ -1,32 +0,0 @@ -# Object files -*.o -*.ko -*.obj -*.elf - -# Precompiled Headers -*.gch -*.pch - -# Libraries -*.lib -*.a -*.la -*.lo - -# Shared objects (inc. Windows DLLs) -*.dll -*.so -*.so.* -*.dylib - -# Executables -*.exe -*.out -*.app -*.i*86 -*.x86_64 -*.hex - -# Debug files -*.dSYM/ diff --git a/cpp/streamvbyte/.travis.yml b/cpp/streamvbyte/.travis.yml deleted file mode 100644 index 6ecd76739..000000000 --- a/cpp/streamvbyte/.travis.yml +++ /dev/null @@ -1,7 +0,0 @@ -language: c -sudo: false -compiler: - - gcc - - clang - -script: make && ./unit diff --git a/cpp/streamvbyte/LICENSE b/cpp/streamvbyte/LICENSE deleted file mode 100644 index e06d20818..000000000 --- a/cpp/streamvbyte/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ -Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/cpp/streamvbyte/README.md b/cpp/streamvbyte/README.md deleted file mode 100644 index 78d16a204..000000000 --- a/cpp/streamvbyte/README.md +++ /dev/null @@ -1,60 +0,0 @@ -streamvbyte -=========== -[![Build Status](https://travis-ci.org/lemire/streamvbyte.png)](https://travis-ci.org/lemire/streamvbyte) - -StreamVByte is a new integer compression technique that applies SIMD instructions (vectorization) to -Google's Group Varint approach. The net result is faster than other byte-oriented compression -techniques. - -The approach is patent-free, the code is available under the Apache License. - - -It includes fast differential coding. - -It assumes a recent Intel processor (e.g., haswell or better) . - -The code should build using most standard-compliant C99 compilers. The provided makefile -expects a Linux-like system. - - -Usage: - - make - ./unit - -See example.c for an example. - -Short code sample: -```C -// suppose that datain is an array of uint32_t integers -size_t compsize = streamvbyte_encode(datain, N, compressedbuffer); // encoding -// here the result is stored in compressedbuffer using compsize bytes -streamvbyte_decode(compressedbuffer, recovdata, N); // decoding (fast) -``` - -If the values are sorted, then it might be preferable to use differential coding: -```C -// suppose that datain is an array of uint32_t integers -size_t compsize = streamvbyte_delta_encode(datain, N, compressedbuffer,0); // encoding -// here the result is stored in compressedbuffer using compsize bytes -streamvbyte_delta_decode(compressedbuffer, recovdata, N,0); // decoding (fast) -``` -You have to know how many integers were coded when you decompress. You can store this -information along with the compressed stream. - -See also --------- -* SIMDCompressionAndIntersection: A C++ library to compress and intersect sorted lists of integers using SIMD instructions https://github.com/lemire/SIMDCompressionAndIntersect -* The FastPFOR C++ library : Fast integer compression https://github.com/lemire/FastPFor -* High-performance dictionary coding https://github.com/lemire/dictionary -* LittleIntPacker: C library to pack and unpack short arrays of integers as fast as possible https://github.com/lemire/LittleIntPacker -* The SIMDComp library: A simple C library for compressing lists of integers using binary packing https://github.com/lemire/simdcomp -* MaskedVByte: Fast decoder for VByte-compressed integers https://github.com/lemire/MaskedVByte -* CSharpFastPFOR: A C# integer compression library https://github.com/Genbox/CSharpFastPFOR -* JavaFastPFOR: A java integer compression library https://github.com/lemire/JavaFastPFOR -* Encoding: Integer Compression Libraries for Go https://github.com/zhenjl/encoding -* FrameOfReference is a C++ library dedicated to frame-of-reference (FOR) compression: https://github.com/lemire/FrameOfReference -* libvbyte: A fast implementation for varbyte 32bit/64bit integer compression https://github.com/cruppstahl/libvbyte -* TurboPFor is a C library that offers lots of interesting optimizations. Well worth checking! (GPL license) https://github.com/powturbo/TurboPFor -* Oroch is a C++ library that offers a usable API (MIT license) https://github.com/ademakov/Oroch - diff --git a/cpp/streamvbyte/example.c b/cpp/streamvbyte/example.c deleted file mode 100644 index 4d72bd1e2..000000000 --- a/cpp/streamvbyte/example.c +++ /dev/null @@ -1,24 +0,0 @@ -#include -#include -#include - -#include "streamvbyte.h" - -int main() { - int N = 5000; - uint32_t * datain = malloc(N * sizeof(uint32_t)); - uint8_t * compressedbuffer = malloc(N * sizeof(uint32_t)); - uint32_t * recovdata = malloc(N * sizeof(uint32_t)); - for (int k = 0; k < N; ++k) - datain[k] = 120; - size_t compsize = streamvbyte_encode(datain, N, compressedbuffer); // encoding - // here the result is stored in compressedbuffer using compsize bytes - size_t compsize2 = streamvbyte_decode(compressedbuffer, recovdata, - N); // decoding (fast) - assert(compsize == compsize2); - free(datain); - free(compressedbuffer); - free(recovdata); - printf("Compressed %d integers down to %d bytes.\n",N,(int) compsize); - return 0; -} diff --git a/cpp/streamvbyte/include/streamvbyte.h b/cpp/streamvbyte/include/streamvbyte.h deleted file mode 100644 index bd4623f6a..000000000 --- a/cpp/streamvbyte/include/streamvbyte.h +++ /dev/null @@ -1,19 +0,0 @@ - -#ifndef VARINTDECODE_H_ -#define VARINTDECODE_H_ -#define __STDC_FORMAT_MACROS -#include -#include // please use a C99-compatible compiler -#include - - -// Encode an array of a given length read from in to bout in varint format. -// Returns the number of bytes written. -size_t streamvbyte_encode(const uint32_t *in, uint32_t length, uint8_t *out); - -// Read "length" 32-bit integers in varint format from in, storing the result in out. -// Returns the number of bytes read. -size_t streamvbyte_decode(const uint8_t* in, uint32_t* out, uint32_t length); - - -#endif /* VARINTDECODE_H_ */ diff --git a/cpp/streamvbyte/include/streamvbytedelta.h b/cpp/streamvbyte/include/streamvbytedelta.h deleted file mode 100644 index 84e47bade..000000000 --- a/cpp/streamvbyte/include/streamvbytedelta.h +++ /dev/null @@ -1,24 +0,0 @@ -/* - * streamvbytedelta.h - * - * Created on: Apr 14, 2016 - * Author: lemire - */ - -#ifndef INCLUDE_STREAMVBYTEDELTA_H_ -#define INCLUDE_STREAMVBYTEDELTA_H_ - - -// Encode an array of a given length read from in to bout in StreamVByte format. -// Returns the number of bytes written. -// this version uses differential coding (coding differences between values) starting at prev (you can often set prev to zero) -size_t streamvbyte_delta_encode(const uint32_t *in, uint32_t length, uint8_t *out, uint32_t prev); - -// Read "length" 32-bit integers in StreamVByte format from in, storing the result in out. -// Returns the number of bytes read. -// this version uses differential coding (coding differences between values) starting at prev (you can often set prev to zero) -size_t streamvbyte_delta_decode(const uint8_t* in, uint32_t* out, uint32_t length, uint32_t prev); - - - -#endif /* INCLUDE_STREAMVBYTEDELTA_H_ */ diff --git a/cpp/streamvbyte/makefile b/cpp/streamvbyte/makefile deleted file mode 100644 index ec44b7ef6..000000000 --- a/cpp/streamvbyte/makefile +++ /dev/null @@ -1,58 +0,0 @@ -# minimalist makefile -.SUFFIXES: -# -.SUFFIXES: .cpp .o .c .h - -CFLAGS = -fPIC -march=native -std=c99 -O3 -Wall -Wextra -pedantic -Wshadow -LDFLAGS = -shared -LIBNAME=libstreamvbyte.so.0.0.1 -all: unit $(LIBNAME) -test: - ./unit -install: $(OBJECTS) - cp $(LIBNAME) /usr/local/lib - ln -s /usr/local/lib/$(LIBNAME) /usr/local/lib/libstreamvbyte.so - ldconfig - cp $(HEADERS) /usr/local/include - - - -HEADERS=./include/streamvbyte.h ./include/streamvbytedelta.h - -uninstall: - for h in $(HEADERS) ; do rm /usr/local/$$h; done - rm /usr/local/lib/$(LIBNAME) - rm /usr/local/lib/libstreamvbyte.so - ldconfig - - -OBJECTS= streamvbyte.o streamvbytedelta.o - - - -streamvbytedelta.o: ./src/streamvbytedelta.c $(HEADERS) - $(CC) $(CFLAGS) -c ./src/streamvbytedelta.c -Iinclude - - -streamvbyte.o: ./src/streamvbyte.c $(HEADERS) - $(CC) $(CFLAGS) -c ./src/streamvbyte.c -Iinclude - - - -$(LIBNAME): $(OBJECTS) - $(CC) $(CFLAGS) -o $(LIBNAME) $(OBJECTS) $(LDFLAGS) - - - - -example: ./example.c $(HEADERS) $(OBJECTS) - $(CC) $(CFLAGS) -o example ./example.c -Iinclude $(OBJECTS) - -unit: ./tests/unit.c $(HEADERS) $(OBJECTS) - $(CC) $(CFLAGS) -o unit ./tests/unit.c -Iinclude $(OBJECTS) - -dynunit: ./tests/unit.c $(HEADERS) $(LIBNAME) - $(CC) $(CFLAGS) -o dynunit ./tests/unit.c -Iinclude -lstreamvbyte - -clean: - rm -f unit *.o $(LIBNAME) example diff --git a/cpp/streamvbyte/src/streamvbyte.c b/cpp/streamvbyte/src/streamvbyte.c deleted file mode 100644 index a45dbf76d..000000000 --- a/cpp/streamvbyte/src/streamvbyte.c +++ /dev/null @@ -1,495 +0,0 @@ -#include "streamvbyte.h" -#if defined(_MSC_VER) - /* Microsoft C/C++-compatible compiler */ - #include -#elif defined(__GNUC__) && (defined(__x86_64__) || defined(__i386__)) - /* GCC-compatible compiler, targeting x86/x86-64 */ - #include -#elif defined(__GNUC__) && defined(__ARM_NEON__) - /* GCC-compatible compiler, targeting ARM with NEON */ - #include -#elif defined(__GNUC__) && defined(__IWMMXT__) - /* GCC-compatible compiler, targeting ARM with WMMX */ - #include -#elif (defined(__GNUC__) || defined(__xlC__)) && (defined(__VEC__) || defined(__ALTIVEC__)) - /* XLC or GCC-compatible compiler, targeting PowerPC with VMX/VSX */ - #include -#elif defined(__GNUC__) && defined(__SPE__) - /* GCC-compatible compiler, targeting PowerPC with SPE */ - #include -#endif - -static uint8_t lengthTable[256] = { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, - 10, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10, 8, 9, 10, 11, 6, 7, 8, 9, 7, 8, - 9, 10, 8, 9, 10, 11, 9, 10, 11, 12, 7, 8, 9, 10, 8, 9, 10, 11, 9, 10, - 11, 12, 10, 11, 12, 13, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10, 8, 9, 10, - 11, 6, 7, 8, 9, 7, 8, 9, 10, 8, 9, 10, 11, 9, 10, 11, 12, 7, 8, 9, 10, - 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 8, 9, 10, 11, 9, 10, 11, - 12, 10, 11, 12, 13, 11, 12, 13, 14, 6, 7, 8, 9, 7, 8, 9, 10, 8, 9, 10, - 11, 9, 10, 11, 12, 7, 8, 9, 10, 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, - 13, 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14, 9, 10, - 11, 12, 10, 11, 12, 13, 11, 12, 13, 14, 12, 13, 14, 15, 7, 8, 9, 10, 8, - 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 8, 9, 10, 11, 9, 10, 11, 12, - 10, 11, 12, 13, 11, 12, 13, 14, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, - 13, 14, 12, 13, 14, 15, 10, 11, 12, 13, 11, 12, 13, 14, 12, 13, 14, 15, - 13, 14, 15, 16 }; - -static uint8_t shuffleTable[256][16] = { { 0, -1, -1, -1, 1, -1, -1, -1, 2, -1, - -1, -1, 3, -1, -1, -1 }, // 1111 - { 0, 1, -1, -1, 2, -1, -1, -1, 3, -1, -1, -1, 4, -1, -1, -1 }, // 2111 - { 0, 1, 2, -1, 3, -1, -1, -1, 4, -1, -1, -1, 5, -1, -1, -1 }, // 3111 - { 0, 1, 2, 3, 4, -1, -1, -1, 5, -1, -1, -1, 6, -1, -1, -1 }, // 4111 - { 0, -1, -1, -1, 1, 2, -1, -1, 3, -1, -1, -1, 4, -1, -1, -1 }, // 1211 - { 0, 1, -1, -1, 2, 3, -1, -1, 4, -1, -1, -1, 5, -1, -1, -1 }, // 2211 - { 0, 1, 2, -1, 3, 4, -1, -1, 5, -1, -1, -1, 6, -1, -1, -1 }, // 3211 - { 0, 1, 2, 3, 4, 5, -1, -1, 6, -1, -1, -1, 7, -1, -1, -1 }, // 4211 - { 0, -1, -1, -1, 1, 2, 3, -1, 4, -1, -1, -1, 5, -1, -1, -1 }, // 1311 - { 0, 1, -1, -1, 2, 3, 4, -1, 5, -1, -1, -1, 6, -1, -1, -1 }, // 2311 - { 0, 1, 2, -1, 3, 4, 5, -1, 6, -1, -1, -1, 7, -1, -1, -1 }, // 3311 - { 0, 1, 2, 3, 4, 5, 6, -1, 7, -1, -1, -1, 8, -1, -1, -1 }, // 4311 - { 0, -1, -1, -1, 1, 2, 3, 4, 5, -1, -1, -1, 6, -1, -1, -1 }, // 1411 - { 0, 1, -1, -1, 2, 3, 4, 5, 6, -1, -1, -1, 7, -1, -1, -1 }, // 2411 - { 0, 1, 2, -1, 3, 4, 5, 6, 7, -1, -1, -1, 8, -1, -1, -1 }, // 3411 - { 0, 1, 2, 3, 4, 5, 6, 7, 8, -1, -1, -1, 9, -1, -1, -1 }, // 4411 - { 0, -1, -1, -1, 1, -1, -1, -1, 2, 3, -1, -1, 4, -1, -1, -1 }, // 1121 - { 0, 1, -1, -1, 2, -1, -1, -1, 3, 4, -1, -1, 5, -1, -1, -1 }, // 2121 - { 0, 1, 2, -1, 3, -1, -1, -1, 4, 5, -1, -1, 6, -1, -1, -1 }, // 3121 - { 0, 1, 2, 3, 4, -1, -1, -1, 5, 6, -1, -1, 7, -1, -1, -1 }, // 4121 - { 0, -1, -1, -1, 1, 2, -1, -1, 3, 4, -1, -1, 5, -1, -1, -1 }, // 1221 - { 0, 1, -1, -1, 2, 3, -1, -1, 4, 5, -1, -1, 6, -1, -1, -1 }, // 2221 - { 0, 1, 2, -1, 3, 4, -1, -1, 5, 6, -1, -1, 7, -1, -1, -1 }, // 3221 - { 0, 1, 2, 3, 4, 5, -1, -1, 6, 7, -1, -1, 8, -1, -1, -1 }, // 4221 - { 0, -1, -1, -1, 1, 2, 3, -1, 4, 5, -1, -1, 6, -1, -1, -1 }, // 1321 - { 0, 1, -1, -1, 2, 3, 4, -1, 5, 6, -1, -1, 7, -1, -1, -1 }, // 2321 - { 0, 1, 2, -1, 3, 4, 5, -1, 6, 7, -1, -1, 8, -1, -1, -1 }, // 3321 - { 0, 1, 2, 3, 4, 5, 6, -1, 7, 8, -1, -1, 9, -1, -1, -1 }, // 4321 - { 0, -1, -1, -1, 1, 2, 3, 4, 5, 6, -1, -1, 7, -1, -1, -1 }, // 1421 - { 0, 1, -1, -1, 2, 3, 4, 5, 6, 7, -1, -1, 8, -1, -1, -1 }, // 2421 - { 0, 1, 2, -1, 3, 4, 5, 6, 7, 8, -1, -1, 9, -1, -1, -1 }, // 3421 - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, 10, -1, -1, -1 }, // 4421 - { 0, -1, -1, -1, 1, -1, -1, -1, 2, 3, 4, -1, 5, -1, -1, -1 }, // 1131 - { 0, 1, -1, -1, 2, -1, -1, -1, 3, 4, 5, -1, 6, -1, -1, -1 }, // 2131 - { 0, 1, 2, -1, 3, -1, -1, -1, 4, 5, 6, -1, 7, -1, -1, -1 }, // 3131 - { 0, 1, 2, 3, 4, -1, -1, -1, 5, 6, 7, -1, 8, -1, -1, -1 }, // 4131 - { 0, -1, -1, -1, 1, 2, -1, -1, 3, 4, 5, -1, 6, -1, -1, -1 }, // 1231 - { 0, 1, -1, -1, 2, 3, -1, -1, 4, 5, 6, -1, 7, -1, -1, -1 }, // 2231 - { 0, 1, 2, -1, 3, 4, -1, -1, 5, 6, 7, -1, 8, -1, -1, -1 }, // 3231 - { 0, 1, 2, 3, 4, 5, -1, -1, 6, 7, 8, -1, 9, -1, -1, -1 }, // 4231 - { 0, -1, -1, -1, 1, 2, 3, -1, 4, 5, 6, -1, 7, -1, -1, -1 }, // 1331 - { 0, 1, -1, -1, 2, 3, 4, -1, 5, 6, 7, -1, 8, -1, -1, -1 }, // 2331 - { 0, 1, 2, -1, 3, 4, 5, -1, 6, 7, 8, -1, 9, -1, -1, -1 }, // 3331 - { 0, 1, 2, 3, 4, 5, 6, -1, 7, 8, 9, -1, 10, -1, -1, -1 }, // 4331 - { 0, -1, -1, -1, 1, 2, 3, 4, 5, 6, 7, -1, 8, -1, -1, -1 }, // 1431 - { 0, 1, -1, -1, 2, 3, 4, 5, 6, 7, 8, -1, 9, -1, -1, -1 }, // 2431 - { 0, 1, 2, -1, 3, 4, 5, 6, 7, 8, 9, -1, 10, -1, -1, -1 }, // 3431 - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, -1, 11, -1, -1, -1 }, // 4431 - { 0, -1, -1, -1, 1, -1, -1, -1, 2, 3, 4, 5, 6, -1, -1, -1 }, // 1141 - { 0, 1, -1, -1, 2, -1, -1, -1, 3, 4, 5, 6, 7, -1, -1, -1 }, // 2141 - { 0, 1, 2, -1, 3, -1, -1, -1, 4, 5, 6, 7, 8, -1, -1, -1 }, // 3141 - { 0, 1, 2, 3, 4, -1, -1, -1, 5, 6, 7, 8, 9, -1, -1, -1 }, // 4141 - { 0, -1, -1, -1, 1, 2, -1, -1, 3, 4, 5, 6, 7, -1, -1, -1 }, // 1241 - { 0, 1, -1, -1, 2, 3, -1, -1, 4, 5, 6, 7, 8, -1, -1, -1 }, // 2241 - { 0, 1, 2, -1, 3, 4, -1, -1, 5, 6, 7, 8, 9, -1, -1, -1 }, // 3241 - { 0, 1, 2, 3, 4, 5, -1, -1, 6, 7, 8, 9, 10, -1, -1, -1 }, // 4241 - { 0, -1, -1, -1, 1, 2, 3, -1, 4, 5, 6, 7, 8, -1, -1, -1 }, // 1341 - { 0, 1, -1, -1, 2, 3, 4, -1, 5, 6, 7, 8, 9, -1, -1, -1 }, // 2341 - { 0, 1, 2, -1, 3, 4, 5, -1, 6, 7, 8, 9, 10, -1, -1, -1 }, // 3341 - { 0, 1, 2, 3, 4, 5, 6, -1, 7, 8, 9, 10, 11, -1, -1, -1 }, // 4341 - { 0, -1, -1, -1, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1 }, // 1441 - { 0, 1, -1, -1, 2, 3, 4, 5, 6, 7, 8, 9, 10, -1, -1, -1 }, // 2441 - { 0, 1, 2, -1, 3, 4, 5, 6, 7, 8, 9, 10, 11, -1, -1, -1 }, // 3441 - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, -1, -1, -1 }, // 4441 - { 0, -1, -1, -1, 1, -1, -1, -1, 2, -1, -1, -1, 3, 4, -1, -1 }, // 1112 - { 0, 1, -1, -1, 2, -1, -1, -1, 3, -1, -1, -1, 4, 5, -1, -1 }, // 2112 - { 0, 1, 2, -1, 3, -1, -1, -1, 4, -1, -1, -1, 5, 6, -1, -1 }, // 3112 - { 0, 1, 2, 3, 4, -1, -1, -1, 5, -1, -1, -1, 6, 7, -1, -1 }, // 4112 - { 0, -1, -1, -1, 1, 2, -1, -1, 3, -1, -1, -1, 4, 5, -1, -1 }, // 1212 - { 0, 1, -1, -1, 2, 3, -1, -1, 4, -1, -1, -1, 5, 6, -1, -1 }, // 2212 - { 0, 1, 2, -1, 3, 4, -1, -1, 5, -1, -1, -1, 6, 7, -1, -1 }, // 3212 - { 0, 1, 2, 3, 4, 5, -1, -1, 6, -1, -1, -1, 7, 8, -1, -1 }, // 4212 - { 0, -1, -1, -1, 1, 2, 3, -1, 4, -1, -1, -1, 5, 6, -1, -1 }, // 1312 - { 0, 1, -1, -1, 2, 3, 4, -1, 5, -1, -1, -1, 6, 7, -1, -1 }, // 2312 - { 0, 1, 2, -1, 3, 4, 5, -1, 6, -1, -1, -1, 7, 8, -1, -1 }, // 3312 - { 0, 1, 2, 3, 4, 5, 6, -1, 7, -1, -1, -1, 8, 9, -1, -1 }, // 4312 - { 0, -1, -1, -1, 1, 2, 3, 4, 5, -1, -1, -1, 6, 7, -1, -1 }, // 1412 - { 0, 1, -1, -1, 2, 3, 4, 5, 6, -1, -1, -1, 7, 8, -1, -1 }, // 2412 - { 0, 1, 2, -1, 3, 4, 5, 6, 7, -1, -1, -1, 8, 9, -1, -1 }, // 3412 - { 0, 1, 2, 3, 4, 5, 6, 7, 8, -1, -1, -1, 9, 10, -1, -1 }, // 4412 - { 0, -1, -1, -1, 1, -1, -1, -1, 2, 3, -1, -1, 4, 5, -1, -1 }, // 1122 - { 0, 1, -1, -1, 2, -1, -1, -1, 3, 4, -1, -1, 5, 6, -1, -1 }, // 2122 - { 0, 1, 2, -1, 3, -1, -1, -1, 4, 5, -1, -1, 6, 7, -1, -1 }, // 3122 - { 0, 1, 2, 3, 4, -1, -1, -1, 5, 6, -1, -1, 7, 8, -1, -1 }, // 4122 - { 0, -1, -1, -1, 1, 2, -1, -1, 3, 4, -1, -1, 5, 6, -1, -1 }, // 1222 - { 0, 1, -1, -1, 2, 3, -1, -1, 4, 5, -1, -1, 6, 7, -1, -1 }, // 2222 - { 0, 1, 2, -1, 3, 4, -1, -1, 5, 6, -1, -1, 7, 8, -1, -1 }, // 3222 - { 0, 1, 2, 3, 4, 5, -1, -1, 6, 7, -1, -1, 8, 9, -1, -1 }, // 4222 - { 0, -1, -1, -1, 1, 2, 3, -1, 4, 5, -1, -1, 6, 7, -1, -1 }, // 1322 - { 0, 1, -1, -1, 2, 3, 4, -1, 5, 6, -1, -1, 7, 8, -1, -1 }, // 2322 - { 0, 1, 2, -1, 3, 4, 5, -1, 6, 7, -1, -1, 8, 9, -1, -1 }, // 3322 - { 0, 1, 2, 3, 4, 5, 6, -1, 7, 8, -1, -1, 9, 10, -1, -1 }, // 4322 - { 0, -1, -1, -1, 1, 2, 3, 4, 5, 6, -1, -1, 7, 8, -1, -1 }, // 1422 - { 0, 1, -1, -1, 2, 3, 4, 5, 6, 7, -1, -1, 8, 9, -1, -1 }, // 2422 - { 0, 1, 2, -1, 3, 4, 5, 6, 7, 8, -1, -1, 9, 10, -1, -1 }, // 3422 - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, 10, 11, -1, -1 }, // 4422 - { 0, -1, -1, -1, 1, -1, -1, -1, 2, 3, 4, -1, 5, 6, -1, -1 }, // 1132 - { 0, 1, -1, -1, 2, -1, -1, -1, 3, 4, 5, -1, 6, 7, -1, -1 }, // 2132 - { 0, 1, 2, -1, 3, -1, -1, -1, 4, 5, 6, -1, 7, 8, -1, -1 }, // 3132 - { 0, 1, 2, 3, 4, -1, -1, -1, 5, 6, 7, -1, 8, 9, -1, -1 }, // 4132 - { 0, -1, -1, -1, 1, 2, -1, -1, 3, 4, 5, -1, 6, 7, -1, -1 }, // 1232 - { 0, 1, -1, -1, 2, 3, -1, -1, 4, 5, 6, -1, 7, 8, -1, -1 }, // 2232 - { 0, 1, 2, -1, 3, 4, -1, -1, 5, 6, 7, -1, 8, 9, -1, -1 }, // 3232 - { 0, 1, 2, 3, 4, 5, -1, -1, 6, 7, 8, -1, 9, 10, -1, -1 }, // 4232 - { 0, -1, -1, -1, 1, 2, 3, -1, 4, 5, 6, -1, 7, 8, -1, -1 }, // 1332 - { 0, 1, -1, -1, 2, 3, 4, -1, 5, 6, 7, -1, 8, 9, -1, -1 }, // 2332 - { 0, 1, 2, -1, 3, 4, 5, -1, 6, 7, 8, -1, 9, 10, -1, -1 }, // 3332 - { 0, 1, 2, 3, 4, 5, 6, -1, 7, 8, 9, -1, 10, 11, -1, -1 }, // 4332 - { 0, -1, -1, -1, 1, 2, 3, 4, 5, 6, 7, -1, 8, 9, -1, -1 }, // 1432 - { 0, 1, -1, -1, 2, 3, 4, 5, 6, 7, 8, -1, 9, 10, -1, -1 }, // 2432 - { 0, 1, 2, -1, 3, 4, 5, 6, 7, 8, 9, -1, 10, 11, -1, -1 }, // 3432 - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, -1, 11, 12, -1, -1 }, // 4432 - { 0, -1, -1, -1, 1, -1, -1, -1, 2, 3, 4, 5, 6, 7, -1, -1 }, // 1142 - { 0, 1, -1, -1, 2, -1, -1, -1, 3, 4, 5, 6, 7, 8, -1, -1 }, // 2142 - { 0, 1, 2, -1, 3, -1, -1, -1, 4, 5, 6, 7, 8, 9, -1, -1 }, // 3142 - { 0, 1, 2, 3, 4, -1, -1, -1, 5, 6, 7, 8, 9, 10, -1, -1 }, // 4142 - { 0, -1, -1, -1, 1, 2, -1, -1, 3, 4, 5, 6, 7, 8, -1, -1 }, // 1242 - { 0, 1, -1, -1, 2, 3, -1, -1, 4, 5, 6, 7, 8, 9, -1, -1 }, // 2242 - { 0, 1, 2, -1, 3, 4, -1, -1, 5, 6, 7, 8, 9, 10, -1, -1 }, // 3242 - { 0, 1, 2, 3, 4, 5, -1, -1, 6, 7, 8, 9, 10, 11, -1, -1 }, // 4242 - { 0, -1, -1, -1, 1, 2, 3, -1, 4, 5, 6, 7, 8, 9, -1, -1 }, // 1342 - { 0, 1, -1, -1, 2, 3, 4, -1, 5, 6, 7, 8, 9, 10, -1, -1 }, // 2342 - { 0, 1, 2, -1, 3, 4, 5, -1, 6, 7, 8, 9, 10, 11, -1, -1 }, // 3342 - { 0, 1, 2, 3, 4, 5, 6, -1, 7, 8, 9, 10, 11, 12, -1, -1 }, // 4342 - { 0, -1, -1, -1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, -1, -1 }, // 1442 - { 0, 1, -1, -1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, -1, -1 }, // 2442 - { 0, 1, 2, -1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, -1, -1 }, // 3442 - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, -1, -1 }, // 4442 - { 0, -1, -1, -1, 1, -1, -1, -1, 2, -1, -1, -1, 3, 4, 5, -1 }, // 1113 - { 0, 1, -1, -1, 2, -1, -1, -1, 3, -1, -1, -1, 4, 5, 6, -1 }, // 2113 - { 0, 1, 2, -1, 3, -1, -1, -1, 4, -1, -1, -1, 5, 6, 7, -1 }, // 3113 - { 0, 1, 2, 3, 4, -1, -1, -1, 5, -1, -1, -1, 6, 7, 8, -1 }, // 4113 - { 0, -1, -1, -1, 1, 2, -1, -1, 3, -1, -1, -1, 4, 5, 6, -1 }, // 1213 - { 0, 1, -1, -1, 2, 3, -1, -1, 4, -1, -1, -1, 5, 6, 7, -1 }, // 2213 - { 0, 1, 2, -1, 3, 4, -1, -1, 5, -1, -1, -1, 6, 7, 8, -1 }, // 3213 - { 0, 1, 2, 3, 4, 5, -1, -1, 6, -1, -1, -1, 7, 8, 9, -1 }, // 4213 - { 0, -1, -1, -1, 1, 2, 3, -1, 4, -1, -1, -1, 5, 6, 7, -1 }, // 1313 - { 0, 1, -1, -1, 2, 3, 4, -1, 5, -1, -1, -1, 6, 7, 8, -1 }, // 2313 - { 0, 1, 2, -1, 3, 4, 5, -1, 6, -1, -1, -1, 7, 8, 9, -1 }, // 3313 - { 0, 1, 2, 3, 4, 5, 6, -1, 7, -1, -1, -1, 8, 9, 10, -1 }, // 4313 - { 0, -1, -1, -1, 1, 2, 3, 4, 5, -1, -1, -1, 6, 7, 8, -1 }, // 1413 - { 0, 1, -1, -1, 2, 3, 4, 5, 6, -1, -1, -1, 7, 8, 9, -1 }, // 2413 - { 0, 1, 2, -1, 3, 4, 5, 6, 7, -1, -1, -1, 8, 9, 10, -1 }, // 3413 - { 0, 1, 2, 3, 4, 5, 6, 7, 8, -1, -1, -1, 9, 10, 11, -1 }, // 4413 - { 0, -1, -1, -1, 1, -1, -1, -1, 2, 3, -1, -1, 4, 5, 6, -1 }, // 1123 - { 0, 1, -1, -1, 2, -1, -1, -1, 3, 4, -1, -1, 5, 6, 7, -1 }, // 2123 - { 0, 1, 2, -1, 3, -1, -1, -1, 4, 5, -1, -1, 6, 7, 8, -1 }, // 3123 - { 0, 1, 2, 3, 4, -1, -1, -1, 5, 6, -1, -1, 7, 8, 9, -1 }, // 4123 - { 0, -1, -1, -1, 1, 2, -1, -1, 3, 4, -1, -1, 5, 6, 7, -1 }, // 1223 - { 0, 1, -1, -1, 2, 3, -1, -1, 4, 5, -1, -1, 6, 7, 8, -1 }, // 2223 - { 0, 1, 2, -1, 3, 4, -1, -1, 5, 6, -1, -1, 7, 8, 9, -1 }, // 3223 - { 0, 1, 2, 3, 4, 5, -1, -1, 6, 7, -1, -1, 8, 9, 10, -1 }, // 4223 - { 0, -1, -1, -1, 1, 2, 3, -1, 4, 5, -1, -1, 6, 7, 8, -1 }, // 1323 - { 0, 1, -1, -1, 2, 3, 4, -1, 5, 6, -1, -1, 7, 8, 9, -1 }, // 2323 - { 0, 1, 2, -1, 3, 4, 5, -1, 6, 7, -1, -1, 8, 9, 10, -1 }, // 3323 - { 0, 1, 2, 3, 4, 5, 6, -1, 7, 8, -1, -1, 9, 10, 11, -1 }, // 4323 - { 0, -1, -1, -1, 1, 2, 3, 4, 5, 6, -1, -1, 7, 8, 9, -1 }, // 1423 - { 0, 1, -1, -1, 2, 3, 4, 5, 6, 7, -1, -1, 8, 9, 10, -1 }, // 2423 - { 0, 1, 2, -1, 3, 4, 5, 6, 7, 8, -1, -1, 9, 10, 11, -1 }, // 3423 - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, 10, 11, 12, -1 }, // 4423 - { 0, -1, -1, -1, 1, -1, -1, -1, 2, 3, 4, -1, 5, 6, 7, -1 }, // 1133 - { 0, 1, -1, -1, 2, -1, -1, -1, 3, 4, 5, -1, 6, 7, 8, -1 }, // 2133 - { 0, 1, 2, -1, 3, -1, -1, -1, 4, 5, 6, -1, 7, 8, 9, -1 }, // 3133 - { 0, 1, 2, 3, 4, -1, -1, -1, 5, 6, 7, -1, 8, 9, 10, -1 }, // 4133 - { 0, -1, -1, -1, 1, 2, -1, -1, 3, 4, 5, -1, 6, 7, 8, -1 }, // 1233 - { 0, 1, -1, -1, 2, 3, -1, -1, 4, 5, 6, -1, 7, 8, 9, -1 }, // 2233 - { 0, 1, 2, -1, 3, 4, -1, -1, 5, 6, 7, -1, 8, 9, 10, -1 }, // 3233 - { 0, 1, 2, 3, 4, 5, -1, -1, 6, 7, 8, -1, 9, 10, 11, -1 }, // 4233 - { 0, -1, -1, -1, 1, 2, 3, -1, 4, 5, 6, -1, 7, 8, 9, -1 }, // 1333 - { 0, 1, -1, -1, 2, 3, 4, -1, 5, 6, 7, -1, 8, 9, 10, -1 }, // 2333 - { 0, 1, 2, -1, 3, 4, 5, -1, 6, 7, 8, -1, 9, 10, 11, -1 }, // 3333 - { 0, 1, 2, 3, 4, 5, 6, -1, 7, 8, 9, -1, 10, 11, 12, -1 }, // 4333 - { 0, -1, -1, -1, 1, 2, 3, 4, 5, 6, 7, -1, 8, 9, 10, -1 }, // 1433 - { 0, 1, -1, -1, 2, 3, 4, 5, 6, 7, 8, -1, 9, 10, 11, -1 }, // 2433 - { 0, 1, 2, -1, 3, 4, 5, 6, 7, 8, 9, -1, 10, 11, 12, -1 }, // 3433 - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, -1, 11, 12, 13, -1 }, // 4433 - { 0, -1, -1, -1, 1, -1, -1, -1, 2, 3, 4, 5, 6, 7, 8, -1 }, // 1143 - { 0, 1, -1, -1, 2, -1, -1, -1, 3, 4, 5, 6, 7, 8, 9, -1 }, // 2143 - { 0, 1, 2, -1, 3, -1, -1, -1, 4, 5, 6, 7, 8, 9, 10, -1 }, // 3143 - { 0, 1, 2, 3, 4, -1, -1, -1, 5, 6, 7, 8, 9, 10, 11, -1 }, // 4143 - { 0, -1, -1, -1, 1, 2, -1, -1, 3, 4, 5, 6, 7, 8, 9, -1 }, // 1243 - { 0, 1, -1, -1, 2, 3, -1, -1, 4, 5, 6, 7, 8, 9, 10, -1 }, // 2243 - { 0, 1, 2, -1, 3, 4, -1, -1, 5, 6, 7, 8, 9, 10, 11, -1 }, // 3243 - { 0, 1, 2, 3, 4, 5, -1, -1, 6, 7, 8, 9, 10, 11, 12, -1 }, // 4243 - { 0, -1, -1, -1, 1, 2, 3, -1, 4, 5, 6, 7, 8, 9, 10, -1 }, // 1343 - { 0, 1, -1, -1, 2, 3, 4, -1, 5, 6, 7, 8, 9, 10, 11, -1 }, // 2343 - { 0, 1, 2, -1, 3, 4, 5, -1, 6, 7, 8, 9, 10, 11, 12, -1 }, // 3343 - { 0, 1, 2, 3, 4, 5, 6, -1, 7, 8, 9, 10, 11, 12, 13, -1 }, // 4343 - { 0, -1, -1, -1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, -1 }, // 1443 - { 0, 1, -1, -1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, -1 }, // 2443 - { 0, 1, 2, -1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, -1 }, // 3443 - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, -1 }, // 4443 - { 0, -1, -1, -1, 1, -1, -1, -1, 2, -1, -1, -1, 3, 4, 5, 6 }, // 1114 - { 0, 1, -1, -1, 2, -1, -1, -1, 3, -1, -1, -1, 4, 5, 6, 7 }, // 2114 - { 0, 1, 2, -1, 3, -1, -1, -1, 4, -1, -1, -1, 5, 6, 7, 8 }, // 3114 - { 0, 1, 2, 3, 4, -1, -1, -1, 5, -1, -1, -1, 6, 7, 8, 9 }, // 4114 - { 0, -1, -1, -1, 1, 2, -1, -1, 3, -1, -1, -1, 4, 5, 6, 7 }, // 1214 - { 0, 1, -1, -1, 2, 3, -1, -1, 4, -1, -1, -1, 5, 6, 7, 8 }, // 2214 - { 0, 1, 2, -1, 3, 4, -1, -1, 5, -1, -1, -1, 6, 7, 8, 9 }, // 3214 - { 0, 1, 2, 3, 4, 5, -1, -1, 6, -1, -1, -1, 7, 8, 9, 10 }, // 4214 - { 0, -1, -1, -1, 1, 2, 3, -1, 4, -1, -1, -1, 5, 6, 7, 8 }, // 1314 - { 0, 1, -1, -1, 2, 3, 4, -1, 5, -1, -1, -1, 6, 7, 8, 9 }, // 2314 - { 0, 1, 2, -1, 3, 4, 5, -1, 6, -1, -1, -1, 7, 8, 9, 10 }, // 3314 - { 0, 1, 2, 3, 4, 5, 6, -1, 7, -1, -1, -1, 8, 9, 10, 11 }, // 4314 - { 0, -1, -1, -1, 1, 2, 3, 4, 5, -1, -1, -1, 6, 7, 8, 9 }, // 1414 - { 0, 1, -1, -1, 2, 3, 4, 5, 6, -1, -1, -1, 7, 8, 9, 10 }, // 2414 - { 0, 1, 2, -1, 3, 4, 5, 6, 7, -1, -1, -1, 8, 9, 10, 11 }, // 3414 - { 0, 1, 2, 3, 4, 5, 6, 7, 8, -1, -1, -1, 9, 10, 11, 12 }, // 4414 - { 0, -1, -1, -1, 1, -1, -1, -1, 2, 3, -1, -1, 4, 5, 6, 7 }, // 1124 - { 0, 1, -1, -1, 2, -1, -1, -1, 3, 4, -1, -1, 5, 6, 7, 8 }, // 2124 - { 0, 1, 2, -1, 3, -1, -1, -1, 4, 5, -1, -1, 6, 7, 8, 9 }, // 3124 - { 0, 1, 2, 3, 4, -1, -1, -1, 5, 6, -1, -1, 7, 8, 9, 10 }, // 4124 - { 0, -1, -1, -1, 1, 2, -1, -1, 3, 4, -1, -1, 5, 6, 7, 8 }, // 1224 - { 0, 1, -1, -1, 2, 3, -1, -1, 4, 5, -1, -1, 6, 7, 8, 9 }, // 2224 - { 0, 1, 2, -1, 3, 4, -1, -1, 5, 6, -1, -1, 7, 8, 9, 10 }, // 3224 - { 0, 1, 2, 3, 4, 5, -1, -1, 6, 7, -1, -1, 8, 9, 10, 11 }, // 4224 - { 0, -1, -1, -1, 1, 2, 3, -1, 4, 5, -1, -1, 6, 7, 8, 9 }, // 1324 - { 0, 1, -1, -1, 2, 3, 4, -1, 5, 6, -1, -1, 7, 8, 9, 10 }, // 2324 - { 0, 1, 2, -1, 3, 4, 5, -1, 6, 7, -1, -1, 8, 9, 10, 11 }, // 3324 - { 0, 1, 2, 3, 4, 5, 6, -1, 7, 8, -1, -1, 9, 10, 11, 12 }, // 4324 - { 0, -1, -1, -1, 1, 2, 3, 4, 5, 6, -1, -1, 7, 8, 9, 10 }, // 1424 - { 0, 1, -1, -1, 2, 3, 4, 5, 6, 7, -1, -1, 8, 9, 10, 11 }, // 2424 - { 0, 1, 2, -1, 3, 4, 5, 6, 7, 8, -1, -1, 9, 10, 11, 12 }, // 3424 - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, 10, 11, 12, 13 }, // 4424 - { 0, -1, -1, -1, 1, -1, -1, -1, 2, 3, 4, -1, 5, 6, 7, 8 }, // 1134 - { 0, 1, -1, -1, 2, -1, -1, -1, 3, 4, 5, -1, 6, 7, 8, 9 }, // 2134 - { 0, 1, 2, -1, 3, -1, -1, -1, 4, 5, 6, -1, 7, 8, 9, 10 }, // 3134 - { 0, 1, 2, 3, 4, -1, -1, -1, 5, 6, 7, -1, 8, 9, 10, 11 }, // 4134 - { 0, -1, -1, -1, 1, 2, -1, -1, 3, 4, 5, -1, 6, 7, 8, 9 }, // 1234 - { 0, 1, -1, -1, 2, 3, -1, -1, 4, 5, 6, -1, 7, 8, 9, 10 }, // 2234 - { 0, 1, 2, -1, 3, 4, -1, -1, 5, 6, 7, -1, 8, 9, 10, 11 }, // 3234 - { 0, 1, 2, 3, 4, 5, -1, -1, 6, 7, 8, -1, 9, 10, 11, 12 }, // 4234 - { 0, -1, -1, -1, 1, 2, 3, -1, 4, 5, 6, -1, 7, 8, 9, 10 }, // 1334 - { 0, 1, -1, -1, 2, 3, 4, -1, 5, 6, 7, -1, 8, 9, 10, 11 }, // 2334 - { 0, 1, 2, -1, 3, 4, 5, -1, 6, 7, 8, -1, 9, 10, 11, 12 }, // 3334 - { 0, 1, 2, 3, 4, 5, 6, -1, 7, 8, 9, -1, 10, 11, 12, 13 }, // 4334 - { 0, -1, -1, -1, 1, 2, 3, 4, 5, 6, 7, -1, 8, 9, 10, 11 }, // 1434 - { 0, 1, -1, -1, 2, 3, 4, 5, 6, 7, 8, -1, 9, 10, 11, 12 }, // 2434 - { 0, 1, 2, -1, 3, 4, 5, 6, 7, 8, 9, -1, 10, 11, 12, 13 }, // 3434 - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, -1, 11, 12, 13, 14 }, // 4434 - { 0, -1, -1, -1, 1, -1, -1, -1, 2, 3, 4, 5, 6, 7, 8, 9 }, // 1144 - { 0, 1, -1, -1, 2, -1, -1, -1, 3, 4, 5, 6, 7, 8, 9, 10 }, // 2144 - { 0, 1, 2, -1, 3, -1, -1, -1, 4, 5, 6, 7, 8, 9, 10, 11 }, // 3144 - { 0, 1, 2, 3, 4, -1, -1, -1, 5, 6, 7, 8, 9, 10, 11, 12 }, // 4144 - { 0, -1, -1, -1, 1, 2, -1, -1, 3, 4, 5, 6, 7, 8, 9, 10 }, // 1244 - { 0, 1, -1, -1, 2, 3, -1, -1, 4, 5, 6, 7, 8, 9, 10, 11 }, // 2244 - { 0, 1, 2, -1, 3, 4, -1, -1, 5, 6, 7, 8, 9, 10, 11, 12 }, // 3244 - { 0, 1, 2, 3, 4, 5, -1, -1, 6, 7, 8, 9, 10, 11, 12, 13 }, // 4244 - { 0, -1, -1, -1, 1, 2, 3, -1, 4, 5, 6, 7, 8, 9, 10, 11 }, // 1344 - { 0, 1, -1, -1, 2, 3, 4, -1, 5, 6, 7, 8, 9, 10, 11, 12 }, // 2344 - { 0, 1, 2, -1, 3, 4, 5, -1, 6, 7, 8, 9, 10, 11, 12, 13 }, // 3344 - { 0, 1, 2, 3, 4, 5, 6, -1, 7, 8, 9, 10, 11, 12, 13, 14 }, // 4344 - { 0, -1, -1, -1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }, // 1444 - { 0, 1, -1, -1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 }, // 2444 - { 0, 1, 2, -1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 }, // 3444 - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } // 4444 -}; - -static uint8_t _encode_data(uint32_t val, uint8_t *__restrict__ *dataPtrPtr) { - uint8_t *dataPtr = *dataPtrPtr; - uint8_t code; - - if (val < (1 << 8)) { // 1 byte - *dataPtr = (uint8_t)(val); - *dataPtrPtr += 1; - code = 0; - } else if (val < (1 << 16)) { // 2 bytes - *(uint16_t *) dataPtr = (uint16_t)(val); - *dataPtrPtr += 2; - code = 1; - } else if (val < (1 << 24)) { // 3 bytes - *(uint16_t *) dataPtr = (uint16_t)(val); - *(dataPtr + 2) = (uint8_t)(val >> 16); - *dataPtrPtr += 3; - code = 2; - } else { // 4 bytes - *(uint32_t *) dataPtr = val; - *dataPtrPtr += 4; - code = 3; - } - - return code; -} - -static uint8_t *svb_encode_scalar(const uint32_t *in, - uint8_t *__restrict__ keyPtr, uint8_t *__restrict__ dataPtr, - uint32_t count) { - if (count == 0) - return dataPtr; // exit immediately if no data - - uint8_t shift = 0; // cycles 0, 2, 4, 6, 0, 2, 4, 6, ... - uint8_t key = 0; - for (uint32_t c = 0; c < count; c++) { - if (shift == 8) { - shift = 0; - *keyPtr++ = key; - key = 0; - } - uint32_t val = in[c]; - uint8_t code = _encode_data(val, &dataPtr); - key |= code << shift; - shift += 2; - } - - *keyPtr = key; // write last key (no increment needed) - return dataPtr; // pointer to first unused data byte -} - -// Encode an array of a given length read from in to bout in streamvbyte format. -// Returns the number of bytes written. -size_t streamvbyte_encode(const uint32_t *in, uint32_t count, uint8_t *out) { - uint8_t *keyPtr = out; - uint32_t keyLen = (count + 3) / 4; // 2-bits rounded to full byte - uint8_t *dataPtr = keyPtr + keyLen; // variable byte data after all keys - return svb_encode_scalar(in, keyPtr, dataPtr, count) - out; -} - -static inline __m128i _decode_avx(uint32_t key, - const uint8_t *__restrict__ *dataPtrPtr) { - uint8_t len = lengthTable[key]; - __m128i Data = _mm_loadu_si128((__m128i *) *dataPtrPtr); - __m128i Shuf = *(__m128i *) &shuffleTable[key]; - - Data = _mm_shuffle_epi8(Data, Shuf); - *dataPtrPtr += len; - return Data; -} - -static inline void _write_avx(uint32_t *out, __m128i Vec) { - _mm_storeu_si128((__m128i *) out, Vec); -} - -static inline uint32_t _decode_data(const uint8_t **dataPtrPtr, uint8_t code) { - const uint8_t *dataPtr = *dataPtrPtr; - uint32_t val; - - if (code == 0) { // 1 byte - val = (uint32_t) * dataPtr; - dataPtr += 1; - } else if (code == 1) { // 2 bytes - val = (uint32_t) * (uint16_t *) dataPtr; - dataPtr += 2; - } else if (code == 2) { // 3 bytes - val = (uint32_t) * (uint16_t *) dataPtr; - val |= *(dataPtr + 2) << 16; - dataPtr += 3; - } else { // code == 3 - val = *(uint32_t *) dataPtr; // 4 bytes - dataPtr += 4; - } - - *dataPtrPtr = dataPtr; - return val; -} -static const uint8_t *svb_decode_scalar(uint32_t *outPtr, const uint8_t *keyPtr, - const uint8_t *dataPtr, uint32_t count) { - if (count == 0) - return dataPtr; // no reads or writes if no data - - uint8_t shift = 0; - uint32_t key = *keyPtr++; - for (uint32_t c = 0; c < count; c++) { - if (shift == 8) { - shift = 0; - key = *keyPtr++; - } - uint32_t val = _decode_data(&dataPtr, (key >> shift) & 0x3); - *outPtr++ = val; - shift += 2; - } - - return dataPtr; // pointer to first unused byte after end -} - -const uint8_t *svb_decode_avx_simple(uint32_t *out, - const uint8_t *__restrict__ keyPtr, const uint8_t *__restrict__ dataPtr, - uint64_t count) { - - uint64_t keybytes = count / 4; // number of key bytes - __m128i Data; - if (keybytes >= 8) { - - int64_t Offset = -(int64_t) keybytes / 8 + 1; - - const uint64_t *keyPtr64 = (const uint64_t *) keyPtr - Offset; - uint64_t nextkeys = keyPtr64[Offset]; - for (; Offset != 0; ++Offset) { - uint64_t keys = nextkeys; - nextkeys = keyPtr64[Offset + 1]; - - Data = _decode_avx((keys & 0xFF), &dataPtr); - _write_avx(out, Data); - Data = _decode_avx((keys & 0xFF00) >> 8, &dataPtr); - _write_avx(out + 4, Data); - - keys >>= 16; - Data = _decode_avx((keys & 0xFF), &dataPtr); - _write_avx(out + 8, Data); - Data = _decode_avx((keys & 0xFF00) >> 8, &dataPtr); - _write_avx(out + 12, Data); - - keys >>= 16; - Data = _decode_avx((keys & 0xFF), &dataPtr); - _write_avx(out + 16, Data); - Data = _decode_avx((keys & 0xFF00) >> 8, &dataPtr); - _write_avx(out + 20, Data); - - keys >>= 16; - Data = _decode_avx((keys & 0xFF), &dataPtr); - _write_avx(out + 24, Data); - Data = _decode_avx((keys & 0xFF00) >> 8, &dataPtr); - _write_avx(out + 28, Data); - - out += 32; - } - { - uint64_t keys = nextkeys; - - Data = _decode_avx((keys & 0xFF), &dataPtr); - _write_avx(out, Data); - Data = _decode_avx((keys & 0xFF00) >> 8, &dataPtr); - _write_avx(out + 4, Data); - - keys >>= 16; - Data = _decode_avx((keys & 0xFF), &dataPtr); - _write_avx(out + 8, Data); - Data = _decode_avx((keys & 0xFF00) >> 8, &dataPtr); - _write_avx(out + 12, Data); - - keys >>= 16; - Data = _decode_avx((keys & 0xFF), &dataPtr); - _write_avx(out + 16, Data); - Data = _decode_avx((keys & 0xFF00) >> 8, &dataPtr); - _write_avx(out + 20, Data); - - keys >>= 16; - Data = _decode_avx((keys & 0xFF), &dataPtr); - _write_avx(out + 24, Data); - Data = _decode_avx((keys & 0xFF00) >> 8, &dataPtr); - _write_avx(out + 28, Data); - - out += 32; - } - } - uint64_t consumedkeys = keybytes - (keybytes & 7); - return svb_decode_scalar(out, keyPtr + consumedkeys, dataPtr, count & 31); -} - -// Read count 32-bit integers in maskedvbyte format from in, storing the result in out. Returns the number of bytes read. -size_t streamvbyte_decode(const uint8_t* in, uint32_t* out, uint32_t count) { - if (count == 0) - return 0; - const uint8_t *keyPtr = in; // full list of keys is next - uint32_t keyLen = ((count + 3) / 4); // 2-bits per key (rounded up) - const uint8_t *dataPtr = keyPtr + keyLen; // data starts at end of keys - return svb_decode_avx_simple(out, keyPtr, dataPtr, count) - in; - -} diff --git a/cpp/streamvbyte/src/streamvbytedelta.c b/cpp/streamvbyte/src/streamvbytedelta.c deleted file mode 100644 index 25e0b7fa0..000000000 --- a/cpp/streamvbyte/src/streamvbytedelta.c +++ /dev/null @@ -1,575 +0,0 @@ -#include "streamvbyte.h" -#if defined(_MSC_VER) - /* Microsoft C/C++-compatible compiler */ - #include -#elif defined(__GNUC__) && (defined(__x86_64__) || defined(__i386__)) - /* GCC-compatible compiler, targeting x86/x86-64 */ - #include -#elif defined(__GNUC__) && defined(__ARM_NEON__) - /* GCC-compatible compiler, targeting ARM with NEON */ - #include -#elif defined(__GNUC__) && defined(__IWMMXT__) - /* GCC-compatible compiler, targeting ARM with WMMX */ - #include -#elif (defined(__GNUC__) || defined(__xlC__)) && (defined(__VEC__) || defined(__ALTIVEC__)) - /* XLC or GCC-compatible compiler, targeting PowerPC with VMX/VSX */ - #include -#elif defined(__GNUC__) && defined(__SPE__) - /* GCC-compatible compiler, targeting PowerPC with SPE */ - #include -#endif - -static uint8_t lengthTable[256] = { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, - 10, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10, 8, 9, 10, 11, 6, 7, 8, 9, 7, 8, - 9, 10, 8, 9, 10, 11, 9, 10, 11, 12, 7, 8, 9, 10, 8, 9, 10, 11, 9, 10, - 11, 12, 10, 11, 12, 13, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10, 8, 9, 10, - 11, 6, 7, 8, 9, 7, 8, 9, 10, 8, 9, 10, 11, 9, 10, 11, 12, 7, 8, 9, 10, - 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 8, 9, 10, 11, 9, 10, 11, - 12, 10, 11, 12, 13, 11, 12, 13, 14, 6, 7, 8, 9, 7, 8, 9, 10, 8, 9, 10, - 11, 9, 10, 11, 12, 7, 8, 9, 10, 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, - 13, 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14, 9, 10, - 11, 12, 10, 11, 12, 13, 11, 12, 13, 14, 12, 13, 14, 15, 7, 8, 9, 10, 8, - 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 8, 9, 10, 11, 9, 10, 11, 12, - 10, 11, 12, 13, 11, 12, 13, 14, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, - 13, 14, 12, 13, 14, 15, 10, 11, 12, 13, 11, 12, 13, 14, 12, 13, 14, 15, - 13, 14, 15, 16 }; - -static uint8_t shuffleTable[256][16] = { { 0, -1, -1, -1, 1, -1, -1, -1, 2, -1, - -1, -1, 3, -1, -1, -1 }, // 1111 - { 0, 1, -1, -1, 2, -1, -1, -1, 3, -1, -1, -1, 4, -1, -1, -1 }, // 2111 - { 0, 1, 2, -1, 3, -1, -1, -1, 4, -1, -1, -1, 5, -1, -1, -1 }, // 3111 - { 0, 1, 2, 3, 4, -1, -1, -1, 5, -1, -1, -1, 6, -1, -1, -1 }, // 4111 - { 0, -1, -1, -1, 1, 2, -1, -1, 3, -1, -1, -1, 4, -1, -1, -1 }, // 1211 - { 0, 1, -1, -1, 2, 3, -1, -1, 4, -1, -1, -1, 5, -1, -1, -1 }, // 2211 - { 0, 1, 2, -1, 3, 4, -1, -1, 5, -1, -1, -1, 6, -1, -1, -1 }, // 3211 - { 0, 1, 2, 3, 4, 5, -1, -1, 6, -1, -1, -1, 7, -1, -1, -1 }, // 4211 - { 0, -1, -1, -1, 1, 2, 3, -1, 4, -1, -1, -1, 5, -1, -1, -1 }, // 1311 - { 0, 1, -1, -1, 2, 3, 4, -1, 5, -1, -1, -1, 6, -1, -1, -1 }, // 2311 - { 0, 1, 2, -1, 3, 4, 5, -1, 6, -1, -1, -1, 7, -1, -1, -1 }, // 3311 - { 0, 1, 2, 3, 4, 5, 6, -1, 7, -1, -1, -1, 8, -1, -1, -1 }, // 4311 - { 0, -1, -1, -1, 1, 2, 3, 4, 5, -1, -1, -1, 6, -1, -1, -1 }, // 1411 - { 0, 1, -1, -1, 2, 3, 4, 5, 6, -1, -1, -1, 7, -1, -1, -1 }, // 2411 - { 0, 1, 2, -1, 3, 4, 5, 6, 7, -1, -1, -1, 8, -1, -1, -1 }, // 3411 - { 0, 1, 2, 3, 4, 5, 6, 7, 8, -1, -1, -1, 9, -1, -1, -1 }, // 4411 - { 0, -1, -1, -1, 1, -1, -1, -1, 2, 3, -1, -1, 4, -1, -1, -1 }, // 1121 - { 0, 1, -1, -1, 2, -1, -1, -1, 3, 4, -1, -1, 5, -1, -1, -1 }, // 2121 - { 0, 1, 2, -1, 3, -1, -1, -1, 4, 5, -1, -1, 6, -1, -1, -1 }, // 3121 - { 0, 1, 2, 3, 4, -1, -1, -1, 5, 6, -1, -1, 7, -1, -1, -1 }, // 4121 - { 0, -1, -1, -1, 1, 2, -1, -1, 3, 4, -1, -1, 5, -1, -1, -1 }, // 1221 - { 0, 1, -1, -1, 2, 3, -1, -1, 4, 5, -1, -1, 6, -1, -1, -1 }, // 2221 - { 0, 1, 2, -1, 3, 4, -1, -1, 5, 6, -1, -1, 7, -1, -1, -1 }, // 3221 - { 0, 1, 2, 3, 4, 5, -1, -1, 6, 7, -1, -1, 8, -1, -1, -1 }, // 4221 - { 0, -1, -1, -1, 1, 2, 3, -1, 4, 5, -1, -1, 6, -1, -1, -1 }, // 1321 - { 0, 1, -1, -1, 2, 3, 4, -1, 5, 6, -1, -1, 7, -1, -1, -1 }, // 2321 - { 0, 1, 2, -1, 3, 4, 5, -1, 6, 7, -1, -1, 8, -1, -1, -1 }, // 3321 - { 0, 1, 2, 3, 4, 5, 6, -1, 7, 8, -1, -1, 9, -1, -1, -1 }, // 4321 - { 0, -1, -1, -1, 1, 2, 3, 4, 5, 6, -1, -1, 7, -1, -1, -1 }, // 1421 - { 0, 1, -1, -1, 2, 3, 4, 5, 6, 7, -1, -1, 8, -1, -1, -1 }, // 2421 - { 0, 1, 2, -1, 3, 4, 5, 6, 7, 8, -1, -1, 9, -1, -1, -1 }, // 3421 - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, 10, -1, -1, -1 }, // 4421 - { 0, -1, -1, -1, 1, -1, -1, -1, 2, 3, 4, -1, 5, -1, -1, -1 }, // 1131 - { 0, 1, -1, -1, 2, -1, -1, -1, 3, 4, 5, -1, 6, -1, -1, -1 }, // 2131 - { 0, 1, 2, -1, 3, -1, -1, -1, 4, 5, 6, -1, 7, -1, -1, -1 }, // 3131 - { 0, 1, 2, 3, 4, -1, -1, -1, 5, 6, 7, -1, 8, -1, -1, -1 }, // 4131 - { 0, -1, -1, -1, 1, 2, -1, -1, 3, 4, 5, -1, 6, -1, -1, -1 }, // 1231 - { 0, 1, -1, -1, 2, 3, -1, -1, 4, 5, 6, -1, 7, -1, -1, -1 }, // 2231 - { 0, 1, 2, -1, 3, 4, -1, -1, 5, 6, 7, -1, 8, -1, -1, -1 }, // 3231 - { 0, 1, 2, 3, 4, 5, -1, -1, 6, 7, 8, -1, 9, -1, -1, -1 }, // 4231 - { 0, -1, -1, -1, 1, 2, 3, -1, 4, 5, 6, -1, 7, -1, -1, -1 }, // 1331 - { 0, 1, -1, -1, 2, 3, 4, -1, 5, 6, 7, -1, 8, -1, -1, -1 }, // 2331 - { 0, 1, 2, -1, 3, 4, 5, -1, 6, 7, 8, -1, 9, -1, -1, -1 }, // 3331 - { 0, 1, 2, 3, 4, 5, 6, -1, 7, 8, 9, -1, 10, -1, -1, -1 }, // 4331 - { 0, -1, -1, -1, 1, 2, 3, 4, 5, 6, 7, -1, 8, -1, -1, -1 }, // 1431 - { 0, 1, -1, -1, 2, 3, 4, 5, 6, 7, 8, -1, 9, -1, -1, -1 }, // 2431 - { 0, 1, 2, -1, 3, 4, 5, 6, 7, 8, 9, -1, 10, -1, -1, -1 }, // 3431 - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, -1, 11, -1, -1, -1 }, // 4431 - { 0, -1, -1, -1, 1, -1, -1, -1, 2, 3, 4, 5, 6, -1, -1, -1 }, // 1141 - { 0, 1, -1, -1, 2, -1, -1, -1, 3, 4, 5, 6, 7, -1, -1, -1 }, // 2141 - { 0, 1, 2, -1, 3, -1, -1, -1, 4, 5, 6, 7, 8, -1, -1, -1 }, // 3141 - { 0, 1, 2, 3, 4, -1, -1, -1, 5, 6, 7, 8, 9, -1, -1, -1 }, // 4141 - { 0, -1, -1, -1, 1, 2, -1, -1, 3, 4, 5, 6, 7, -1, -1, -1 }, // 1241 - { 0, 1, -1, -1, 2, 3, -1, -1, 4, 5, 6, 7, 8, -1, -1, -1 }, // 2241 - { 0, 1, 2, -1, 3, 4, -1, -1, 5, 6, 7, 8, 9, -1, -1, -1 }, // 3241 - { 0, 1, 2, 3, 4, 5, -1, -1, 6, 7, 8, 9, 10, -1, -1, -1 }, // 4241 - { 0, -1, -1, -1, 1, 2, 3, -1, 4, 5, 6, 7, 8, -1, -1, -1 }, // 1341 - { 0, 1, -1, -1, 2, 3, 4, -1, 5, 6, 7, 8, 9, -1, -1, -1 }, // 2341 - { 0, 1, 2, -1, 3, 4, 5, -1, 6, 7, 8, 9, 10, -1, -1, -1 }, // 3341 - { 0, 1, 2, 3, 4, 5, 6, -1, 7, 8, 9, 10, 11, -1, -1, -1 }, // 4341 - { 0, -1, -1, -1, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1 }, // 1441 - { 0, 1, -1, -1, 2, 3, 4, 5, 6, 7, 8, 9, 10, -1, -1, -1 }, // 2441 - { 0, 1, 2, -1, 3, 4, 5, 6, 7, 8, 9, 10, 11, -1, -1, -1 }, // 3441 - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, -1, -1, -1 }, // 4441 - { 0, -1, -1, -1, 1, -1, -1, -1, 2, -1, -1, -1, 3, 4, -1, -1 }, // 1112 - { 0, 1, -1, -1, 2, -1, -1, -1, 3, -1, -1, -1, 4, 5, -1, -1 }, // 2112 - { 0, 1, 2, -1, 3, -1, -1, -1, 4, -1, -1, -1, 5, 6, -1, -1 }, // 3112 - { 0, 1, 2, 3, 4, -1, -1, -1, 5, -1, -1, -1, 6, 7, -1, -1 }, // 4112 - { 0, -1, -1, -1, 1, 2, -1, -1, 3, -1, -1, -1, 4, 5, -1, -1 }, // 1212 - { 0, 1, -1, -1, 2, 3, -1, -1, 4, -1, -1, -1, 5, 6, -1, -1 }, // 2212 - { 0, 1, 2, -1, 3, 4, -1, -1, 5, -1, -1, -1, 6, 7, -1, -1 }, // 3212 - { 0, 1, 2, 3, 4, 5, -1, -1, 6, -1, -1, -1, 7, 8, -1, -1 }, // 4212 - { 0, -1, -1, -1, 1, 2, 3, -1, 4, -1, -1, -1, 5, 6, -1, -1 }, // 1312 - { 0, 1, -1, -1, 2, 3, 4, -1, 5, -1, -1, -1, 6, 7, -1, -1 }, // 2312 - { 0, 1, 2, -1, 3, 4, 5, -1, 6, -1, -1, -1, 7, 8, -1, -1 }, // 3312 - { 0, 1, 2, 3, 4, 5, 6, -1, 7, -1, -1, -1, 8, 9, -1, -1 }, // 4312 - { 0, -1, -1, -1, 1, 2, 3, 4, 5, -1, -1, -1, 6, 7, -1, -1 }, // 1412 - { 0, 1, -1, -1, 2, 3, 4, 5, 6, -1, -1, -1, 7, 8, -1, -1 }, // 2412 - { 0, 1, 2, -1, 3, 4, 5, 6, 7, -1, -1, -1, 8, 9, -1, -1 }, // 3412 - { 0, 1, 2, 3, 4, 5, 6, 7, 8, -1, -1, -1, 9, 10, -1, -1 }, // 4412 - { 0, -1, -1, -1, 1, -1, -1, -1, 2, 3, -1, -1, 4, 5, -1, -1 }, // 1122 - { 0, 1, -1, -1, 2, -1, -1, -1, 3, 4, -1, -1, 5, 6, -1, -1 }, // 2122 - { 0, 1, 2, -1, 3, -1, -1, -1, 4, 5, -1, -1, 6, 7, -1, -1 }, // 3122 - { 0, 1, 2, 3, 4, -1, -1, -1, 5, 6, -1, -1, 7, 8, -1, -1 }, // 4122 - { 0, -1, -1, -1, 1, 2, -1, -1, 3, 4, -1, -1, 5, 6, -1, -1 }, // 1222 - { 0, 1, -1, -1, 2, 3, -1, -1, 4, 5, -1, -1, 6, 7, -1, -1 }, // 2222 - { 0, 1, 2, -1, 3, 4, -1, -1, 5, 6, -1, -1, 7, 8, -1, -1 }, // 3222 - { 0, 1, 2, 3, 4, 5, -1, -1, 6, 7, -1, -1, 8, 9, -1, -1 }, // 4222 - { 0, -1, -1, -1, 1, 2, 3, -1, 4, 5, -1, -1, 6, 7, -1, -1 }, // 1322 - { 0, 1, -1, -1, 2, 3, 4, -1, 5, 6, -1, -1, 7, 8, -1, -1 }, // 2322 - { 0, 1, 2, -1, 3, 4, 5, -1, 6, 7, -1, -1, 8, 9, -1, -1 }, // 3322 - { 0, 1, 2, 3, 4, 5, 6, -1, 7, 8, -1, -1, 9, 10, -1, -1 }, // 4322 - { 0, -1, -1, -1, 1, 2, 3, 4, 5, 6, -1, -1, 7, 8, -1, -1 }, // 1422 - { 0, 1, -1, -1, 2, 3, 4, 5, 6, 7, -1, -1, 8, 9, -1, -1 }, // 2422 - { 0, 1, 2, -1, 3, 4, 5, 6, 7, 8, -1, -1, 9, 10, -1, -1 }, // 3422 - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, 10, 11, -1, -1 }, // 4422 - { 0, -1, -1, -1, 1, -1, -1, -1, 2, 3, 4, -1, 5, 6, -1, -1 }, // 1132 - { 0, 1, -1, -1, 2, -1, -1, -1, 3, 4, 5, -1, 6, 7, -1, -1 }, // 2132 - { 0, 1, 2, -1, 3, -1, -1, -1, 4, 5, 6, -1, 7, 8, -1, -1 }, // 3132 - { 0, 1, 2, 3, 4, -1, -1, -1, 5, 6, 7, -1, 8, 9, -1, -1 }, // 4132 - { 0, -1, -1, -1, 1, 2, -1, -1, 3, 4, 5, -1, 6, 7, -1, -1 }, // 1232 - { 0, 1, -1, -1, 2, 3, -1, -1, 4, 5, 6, -1, 7, 8, -1, -1 }, // 2232 - { 0, 1, 2, -1, 3, 4, -1, -1, 5, 6, 7, -1, 8, 9, -1, -1 }, // 3232 - { 0, 1, 2, 3, 4, 5, -1, -1, 6, 7, 8, -1, 9, 10, -1, -1 }, // 4232 - { 0, -1, -1, -1, 1, 2, 3, -1, 4, 5, 6, -1, 7, 8, -1, -1 }, // 1332 - { 0, 1, -1, -1, 2, 3, 4, -1, 5, 6, 7, -1, 8, 9, -1, -1 }, // 2332 - { 0, 1, 2, -1, 3, 4, 5, -1, 6, 7, 8, -1, 9, 10, -1, -1 }, // 3332 - { 0, 1, 2, 3, 4, 5, 6, -1, 7, 8, 9, -1, 10, 11, -1, -1 }, // 4332 - { 0, -1, -1, -1, 1, 2, 3, 4, 5, 6, 7, -1, 8, 9, -1, -1 }, // 1432 - { 0, 1, -1, -1, 2, 3, 4, 5, 6, 7, 8, -1, 9, 10, -1, -1 }, // 2432 - { 0, 1, 2, -1, 3, 4, 5, 6, 7, 8, 9, -1, 10, 11, -1, -1 }, // 3432 - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, -1, 11, 12, -1, -1 }, // 4432 - { 0, -1, -1, -1, 1, -1, -1, -1, 2, 3, 4, 5, 6, 7, -1, -1 }, // 1142 - { 0, 1, -1, -1, 2, -1, -1, -1, 3, 4, 5, 6, 7, 8, -1, -1 }, // 2142 - { 0, 1, 2, -1, 3, -1, -1, -1, 4, 5, 6, 7, 8, 9, -1, -1 }, // 3142 - { 0, 1, 2, 3, 4, -1, -1, -1, 5, 6, 7, 8, 9, 10, -1, -1 }, // 4142 - { 0, -1, -1, -1, 1, 2, -1, -1, 3, 4, 5, 6, 7, 8, -1, -1 }, // 1242 - { 0, 1, -1, -1, 2, 3, -1, -1, 4, 5, 6, 7, 8, 9, -1, -1 }, // 2242 - { 0, 1, 2, -1, 3, 4, -1, -1, 5, 6, 7, 8, 9, 10, -1, -1 }, // 3242 - { 0, 1, 2, 3, 4, 5, -1, -1, 6, 7, 8, 9, 10, 11, -1, -1 }, // 4242 - { 0, -1, -1, -1, 1, 2, 3, -1, 4, 5, 6, 7, 8, 9, -1, -1 }, // 1342 - { 0, 1, -1, -1, 2, 3, 4, -1, 5, 6, 7, 8, 9, 10, -1, -1 }, // 2342 - { 0, 1, 2, -1, 3, 4, 5, -1, 6, 7, 8, 9, 10, 11, -1, -1 }, // 3342 - { 0, 1, 2, 3, 4, 5, 6, -1, 7, 8, 9, 10, 11, 12, -1, -1 }, // 4342 - { 0, -1, -1, -1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, -1, -1 }, // 1442 - { 0, 1, -1, -1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, -1, -1 }, // 2442 - { 0, 1, 2, -1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, -1, -1 }, // 3442 - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, -1, -1 }, // 4442 - { 0, -1, -1, -1, 1, -1, -1, -1, 2, -1, -1, -1, 3, 4, 5, -1 }, // 1113 - { 0, 1, -1, -1, 2, -1, -1, -1, 3, -1, -1, -1, 4, 5, 6, -1 }, // 2113 - { 0, 1, 2, -1, 3, -1, -1, -1, 4, -1, -1, -1, 5, 6, 7, -1 }, // 3113 - { 0, 1, 2, 3, 4, -1, -1, -1, 5, -1, -1, -1, 6, 7, 8, -1 }, // 4113 - { 0, -1, -1, -1, 1, 2, -1, -1, 3, -1, -1, -1, 4, 5, 6, -1 }, // 1213 - { 0, 1, -1, -1, 2, 3, -1, -1, 4, -1, -1, -1, 5, 6, 7, -1 }, // 2213 - { 0, 1, 2, -1, 3, 4, -1, -1, 5, -1, -1, -1, 6, 7, 8, -1 }, // 3213 - { 0, 1, 2, 3, 4, 5, -1, -1, 6, -1, -1, -1, 7, 8, 9, -1 }, // 4213 - { 0, -1, -1, -1, 1, 2, 3, -1, 4, -1, -1, -1, 5, 6, 7, -1 }, // 1313 - { 0, 1, -1, -1, 2, 3, 4, -1, 5, -1, -1, -1, 6, 7, 8, -1 }, // 2313 - { 0, 1, 2, -1, 3, 4, 5, -1, 6, -1, -1, -1, 7, 8, 9, -1 }, // 3313 - { 0, 1, 2, 3, 4, 5, 6, -1, 7, -1, -1, -1, 8, 9, 10, -1 }, // 4313 - { 0, -1, -1, -1, 1, 2, 3, 4, 5, -1, -1, -1, 6, 7, 8, -1 }, // 1413 - { 0, 1, -1, -1, 2, 3, 4, 5, 6, -1, -1, -1, 7, 8, 9, -1 }, // 2413 - { 0, 1, 2, -1, 3, 4, 5, 6, 7, -1, -1, -1, 8, 9, 10, -1 }, // 3413 - { 0, 1, 2, 3, 4, 5, 6, 7, 8, -1, -1, -1, 9, 10, 11, -1 }, // 4413 - { 0, -1, -1, -1, 1, -1, -1, -1, 2, 3, -1, -1, 4, 5, 6, -1 }, // 1123 - { 0, 1, -1, -1, 2, -1, -1, -1, 3, 4, -1, -1, 5, 6, 7, -1 }, // 2123 - { 0, 1, 2, -1, 3, -1, -1, -1, 4, 5, -1, -1, 6, 7, 8, -1 }, // 3123 - { 0, 1, 2, 3, 4, -1, -1, -1, 5, 6, -1, -1, 7, 8, 9, -1 }, // 4123 - { 0, -1, -1, -1, 1, 2, -1, -1, 3, 4, -1, -1, 5, 6, 7, -1 }, // 1223 - { 0, 1, -1, -1, 2, 3, -1, -1, 4, 5, -1, -1, 6, 7, 8, -1 }, // 2223 - { 0, 1, 2, -1, 3, 4, -1, -1, 5, 6, -1, -1, 7, 8, 9, -1 }, // 3223 - { 0, 1, 2, 3, 4, 5, -1, -1, 6, 7, -1, -1, 8, 9, 10, -1 }, // 4223 - { 0, -1, -1, -1, 1, 2, 3, -1, 4, 5, -1, -1, 6, 7, 8, -1 }, // 1323 - { 0, 1, -1, -1, 2, 3, 4, -1, 5, 6, -1, -1, 7, 8, 9, -1 }, // 2323 - { 0, 1, 2, -1, 3, 4, 5, -1, 6, 7, -1, -1, 8, 9, 10, -1 }, // 3323 - { 0, 1, 2, 3, 4, 5, 6, -1, 7, 8, -1, -1, 9, 10, 11, -1 }, // 4323 - { 0, -1, -1, -1, 1, 2, 3, 4, 5, 6, -1, -1, 7, 8, 9, -1 }, // 1423 - { 0, 1, -1, -1, 2, 3, 4, 5, 6, 7, -1, -1, 8, 9, 10, -1 }, // 2423 - { 0, 1, 2, -1, 3, 4, 5, 6, 7, 8, -1, -1, 9, 10, 11, -1 }, // 3423 - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, 10, 11, 12, -1 }, // 4423 - { 0, -1, -1, -1, 1, -1, -1, -1, 2, 3, 4, -1, 5, 6, 7, -1 }, // 1133 - { 0, 1, -1, -1, 2, -1, -1, -1, 3, 4, 5, -1, 6, 7, 8, -1 }, // 2133 - { 0, 1, 2, -1, 3, -1, -1, -1, 4, 5, 6, -1, 7, 8, 9, -1 }, // 3133 - { 0, 1, 2, 3, 4, -1, -1, -1, 5, 6, 7, -1, 8, 9, 10, -1 }, // 4133 - { 0, -1, -1, -1, 1, 2, -1, -1, 3, 4, 5, -1, 6, 7, 8, -1 }, // 1233 - { 0, 1, -1, -1, 2, 3, -1, -1, 4, 5, 6, -1, 7, 8, 9, -1 }, // 2233 - { 0, 1, 2, -1, 3, 4, -1, -1, 5, 6, 7, -1, 8, 9, 10, -1 }, // 3233 - { 0, 1, 2, 3, 4, 5, -1, -1, 6, 7, 8, -1, 9, 10, 11, -1 }, // 4233 - { 0, -1, -1, -1, 1, 2, 3, -1, 4, 5, 6, -1, 7, 8, 9, -1 }, // 1333 - { 0, 1, -1, -1, 2, 3, 4, -1, 5, 6, 7, -1, 8, 9, 10, -1 }, // 2333 - { 0, 1, 2, -1, 3, 4, 5, -1, 6, 7, 8, -1, 9, 10, 11, -1 }, // 3333 - { 0, 1, 2, 3, 4, 5, 6, -1, 7, 8, 9, -1, 10, 11, 12, -1 }, // 4333 - { 0, -1, -1, -1, 1, 2, 3, 4, 5, 6, 7, -1, 8, 9, 10, -1 }, // 1433 - { 0, 1, -1, -1, 2, 3, 4, 5, 6, 7, 8, -1, 9, 10, 11, -1 }, // 2433 - { 0, 1, 2, -1, 3, 4, 5, 6, 7, 8, 9, -1, 10, 11, 12, -1 }, // 3433 - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, -1, 11, 12, 13, -1 }, // 4433 - { 0, -1, -1, -1, 1, -1, -1, -1, 2, 3, 4, 5, 6, 7, 8, -1 }, // 1143 - { 0, 1, -1, -1, 2, -1, -1, -1, 3, 4, 5, 6, 7, 8, 9, -1 }, // 2143 - { 0, 1, 2, -1, 3, -1, -1, -1, 4, 5, 6, 7, 8, 9, 10, -1 }, // 3143 - { 0, 1, 2, 3, 4, -1, -1, -1, 5, 6, 7, 8, 9, 10, 11, -1 }, // 4143 - { 0, -1, -1, -1, 1, 2, -1, -1, 3, 4, 5, 6, 7, 8, 9, -1 }, // 1243 - { 0, 1, -1, -1, 2, 3, -1, -1, 4, 5, 6, 7, 8, 9, 10, -1 }, // 2243 - { 0, 1, 2, -1, 3, 4, -1, -1, 5, 6, 7, 8, 9, 10, 11, -1 }, // 3243 - { 0, 1, 2, 3, 4, 5, -1, -1, 6, 7, 8, 9, 10, 11, 12, -1 }, // 4243 - { 0, -1, -1, -1, 1, 2, 3, -1, 4, 5, 6, 7, 8, 9, 10, -1 }, // 1343 - { 0, 1, -1, -1, 2, 3, 4, -1, 5, 6, 7, 8, 9, 10, 11, -1 }, // 2343 - { 0, 1, 2, -1, 3, 4, 5, -1, 6, 7, 8, 9, 10, 11, 12, -1 }, // 3343 - { 0, 1, 2, 3, 4, 5, 6, -1, 7, 8, 9, 10, 11, 12, 13, -1 }, // 4343 - { 0, -1, -1, -1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, -1 }, // 1443 - { 0, 1, -1, -1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, -1 }, // 2443 - { 0, 1, 2, -1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, -1 }, // 3443 - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, -1 }, // 4443 - { 0, -1, -1, -1, 1, -1, -1, -1, 2, -1, -1, -1, 3, 4, 5, 6 }, // 1114 - { 0, 1, -1, -1, 2, -1, -1, -1, 3, -1, -1, -1, 4, 5, 6, 7 }, // 2114 - { 0, 1, 2, -1, 3, -1, -1, -1, 4, -1, -1, -1, 5, 6, 7, 8 }, // 3114 - { 0, 1, 2, 3, 4, -1, -1, -1, 5, -1, -1, -1, 6, 7, 8, 9 }, // 4114 - { 0, -1, -1, -1, 1, 2, -1, -1, 3, -1, -1, -1, 4, 5, 6, 7 }, // 1214 - { 0, 1, -1, -1, 2, 3, -1, -1, 4, -1, -1, -1, 5, 6, 7, 8 }, // 2214 - { 0, 1, 2, -1, 3, 4, -1, -1, 5, -1, -1, -1, 6, 7, 8, 9 }, // 3214 - { 0, 1, 2, 3, 4, 5, -1, -1, 6, -1, -1, -1, 7, 8, 9, 10 }, // 4214 - { 0, -1, -1, -1, 1, 2, 3, -1, 4, -1, -1, -1, 5, 6, 7, 8 }, // 1314 - { 0, 1, -1, -1, 2, 3, 4, -1, 5, -1, -1, -1, 6, 7, 8, 9 }, // 2314 - { 0, 1, 2, -1, 3, 4, 5, -1, 6, -1, -1, -1, 7, 8, 9, 10 }, // 3314 - { 0, 1, 2, 3, 4, 5, 6, -1, 7, -1, -1, -1, 8, 9, 10, 11 }, // 4314 - { 0, -1, -1, -1, 1, 2, 3, 4, 5, -1, -1, -1, 6, 7, 8, 9 }, // 1414 - { 0, 1, -1, -1, 2, 3, 4, 5, 6, -1, -1, -1, 7, 8, 9, 10 }, // 2414 - { 0, 1, 2, -1, 3, 4, 5, 6, 7, -1, -1, -1, 8, 9, 10, 11 }, // 3414 - { 0, 1, 2, 3, 4, 5, 6, 7, 8, -1, -1, -1, 9, 10, 11, 12 }, // 4414 - { 0, -1, -1, -1, 1, -1, -1, -1, 2, 3, -1, -1, 4, 5, 6, 7 }, // 1124 - { 0, 1, -1, -1, 2, -1, -1, -1, 3, 4, -1, -1, 5, 6, 7, 8 }, // 2124 - { 0, 1, 2, -1, 3, -1, -1, -1, 4, 5, -1, -1, 6, 7, 8, 9 }, // 3124 - { 0, 1, 2, 3, 4, -1, -1, -1, 5, 6, -1, -1, 7, 8, 9, 10 }, // 4124 - { 0, -1, -1, -1, 1, 2, -1, -1, 3, 4, -1, -1, 5, 6, 7, 8 }, // 1224 - { 0, 1, -1, -1, 2, 3, -1, -1, 4, 5, -1, -1, 6, 7, 8, 9 }, // 2224 - { 0, 1, 2, -1, 3, 4, -1, -1, 5, 6, -1, -1, 7, 8, 9, 10 }, // 3224 - { 0, 1, 2, 3, 4, 5, -1, -1, 6, 7, -1, -1, 8, 9, 10, 11 }, // 4224 - { 0, -1, -1, -1, 1, 2, 3, -1, 4, 5, -1, -1, 6, 7, 8, 9 }, // 1324 - { 0, 1, -1, -1, 2, 3, 4, -1, 5, 6, -1, -1, 7, 8, 9, 10 }, // 2324 - { 0, 1, 2, -1, 3, 4, 5, -1, 6, 7, -1, -1, 8, 9, 10, 11 }, // 3324 - { 0, 1, 2, 3, 4, 5, 6, -1, 7, 8, -1, -1, 9, 10, 11, 12 }, // 4324 - { 0, -1, -1, -1, 1, 2, 3, 4, 5, 6, -1, -1, 7, 8, 9, 10 }, // 1424 - { 0, 1, -1, -1, 2, 3, 4, 5, 6, 7, -1, -1, 8, 9, 10, 11 }, // 2424 - { 0, 1, 2, -1, 3, 4, 5, 6, 7, 8, -1, -1, 9, 10, 11, 12 }, // 3424 - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, 10, 11, 12, 13 }, // 4424 - { 0, -1, -1, -1, 1, -1, -1, -1, 2, 3, 4, -1, 5, 6, 7, 8 }, // 1134 - { 0, 1, -1, -1, 2, -1, -1, -1, 3, 4, 5, -1, 6, 7, 8, 9 }, // 2134 - { 0, 1, 2, -1, 3, -1, -1, -1, 4, 5, 6, -1, 7, 8, 9, 10 }, // 3134 - { 0, 1, 2, 3, 4, -1, -1, -1, 5, 6, 7, -1, 8, 9, 10, 11 }, // 4134 - { 0, -1, -1, -1, 1, 2, -1, -1, 3, 4, 5, -1, 6, 7, 8, 9 }, // 1234 - { 0, 1, -1, -1, 2, 3, -1, -1, 4, 5, 6, -1, 7, 8, 9, 10 }, // 2234 - { 0, 1, 2, -1, 3, 4, -1, -1, 5, 6, 7, -1, 8, 9, 10, 11 }, // 3234 - { 0, 1, 2, 3, 4, 5, -1, -1, 6, 7, 8, -1, 9, 10, 11, 12 }, // 4234 - { 0, -1, -1, -1, 1, 2, 3, -1, 4, 5, 6, -1, 7, 8, 9, 10 }, // 1334 - { 0, 1, -1, -1, 2, 3, 4, -1, 5, 6, 7, -1, 8, 9, 10, 11 }, // 2334 - { 0, 1, 2, -1, 3, 4, 5, -1, 6, 7, 8, -1, 9, 10, 11, 12 }, // 3334 - { 0, 1, 2, 3, 4, 5, 6, -1, 7, 8, 9, -1, 10, 11, 12, 13 }, // 4334 - { 0, -1, -1, -1, 1, 2, 3, 4, 5, 6, 7, -1, 8, 9, 10, 11 }, // 1434 - { 0, 1, -1, -1, 2, 3, 4, 5, 6, 7, 8, -1, 9, 10, 11, 12 }, // 2434 - { 0, 1, 2, -1, 3, 4, 5, 6, 7, 8, 9, -1, 10, 11, 12, 13 }, // 3434 - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, -1, 11, 12, 13, 14 }, // 4434 - { 0, -1, -1, -1, 1, -1, -1, -1, 2, 3, 4, 5, 6, 7, 8, 9 }, // 1144 - { 0, 1, -1, -1, 2, -1, -1, -1, 3, 4, 5, 6, 7, 8, 9, 10 }, // 2144 - { 0, 1, 2, -1, 3, -1, -1, -1, 4, 5, 6, 7, 8, 9, 10, 11 }, // 3144 - { 0, 1, 2, 3, 4, -1, -1, -1, 5, 6, 7, 8, 9, 10, 11, 12 }, // 4144 - { 0, -1, -1, -1, 1, 2, -1, -1, 3, 4, 5, 6, 7, 8, 9, 10 }, // 1244 - { 0, 1, -1, -1, 2, 3, -1, -1, 4, 5, 6, 7, 8, 9, 10, 11 }, // 2244 - { 0, 1, 2, -1, 3, 4, -1, -1, 5, 6, 7, 8, 9, 10, 11, 12 }, // 3244 - { 0, 1, 2, 3, 4, 5, -1, -1, 6, 7, 8, 9, 10, 11, 12, 13 }, // 4244 - { 0, -1, -1, -1, 1, 2, 3, -1, 4, 5, 6, 7, 8, 9, 10, 11 }, // 1344 - { 0, 1, -1, -1, 2, 3, 4, -1, 5, 6, 7, 8, 9, 10, 11, 12 }, // 2344 - { 0, 1, 2, -1, 3, 4, 5, -1, 6, 7, 8, 9, 10, 11, 12, 13 }, // 3344 - { 0, 1, 2, 3, 4, 5, 6, -1, 7, 8, 9, 10, 11, 12, 13, 14 }, // 4344 - { 0, -1, -1, -1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }, // 1444 - { 0, 1, -1, -1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 }, // 2444 - { 0, 1, 2, -1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 }, // 3444 - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } // 4444 -}; - -static uint8_t _encode_data(uint32_t val, uint8_t *__restrict__ *dataPtrPtr) { - uint8_t *dataPtr = *dataPtrPtr; - uint8_t code; - - if (val < (1 << 8)) { // 1 byte - *dataPtr = (uint8_t)(val); - *dataPtrPtr += 1; - code = 0; - } else if (val < (1 << 16)) { // 2 bytes - *(uint16_t *) dataPtr = (uint16_t)(val); - *dataPtrPtr += 2; - code = 1; - } else if (val < (1 << 24)) { // 3 bytes - *(uint16_t *) dataPtr = (uint16_t)(val); - *(dataPtr + 2) = (uint8_t)(val >> 16); - *dataPtrPtr += 3; - code = 2; - } else { // 4 bytes - *(uint32_t *) dataPtr = val; - *dataPtrPtr += 4; - code = 3; - } - - return code; -} - -static uint8_t *svb_encode_scalar_d1_init(const uint32_t *in, - uint8_t *__restrict__ keyPtr, uint8_t *__restrict__ dataPtr, - uint32_t count, uint32_t prev) { - if (count == 0) - return dataPtr; // exit immediately if no data - - uint8_t shift = 0; // cycles 0, 2, 4, 6, 0, 2, 4, 6, ... - uint8_t key = 0; - for (uint32_t c = 0; c < count; c++) { - if (shift == 8) { - shift = 0; - *keyPtr++ = key; - key = 0; - } - uint32_t val = in[c] - prev; - prev = in[c]; - uint8_t code = _encode_data(val, &dataPtr); - key |= code << shift; - shift += 2; - } - - *keyPtr = key; // write last key (no increment needed) - return dataPtr; // pointer to first unused data byte -} - -size_t streamvbyte_delta_encode(const uint32_t *in, uint32_t count, uint8_t *out, - uint32_t prev) { - uint8_t *keyPtr = out; // keys come immediately after 32-bit count - uint32_t keyLen = (count + 3) / 4; // 2-bits rounded to full byte - uint8_t *dataPtr = keyPtr + keyLen; // variable byte data after all keys - - return svb_encode_scalar_d1_init(in, keyPtr, dataPtr, count, prev) - out; - -} - -static inline __m128i _decode_avx(uint32_t key, const uint8_t *__restrict__ *dataPtrPtr) { - uint8_t len = lengthTable[key]; - __m128i Data = _mm_loadu_si128((__m128i *) *dataPtrPtr); - __m128i Shuf = *(__m128i *) &shuffleTable[key]; - - Data = _mm_shuffle_epi8(Data, Shuf); - *dataPtrPtr += len; - - return Data; -} -#define BroadcastLastXMM 0xFF // bits 0-7 all set to choose highest element - - - -static inline void _write_avx(uint32_t *out, __m128i Vec) { - _mm_storeu_si128((__m128i *) out, Vec); -} - -static __m128i _write_avx_d1(uint32_t *out, __m128i Vec, __m128i Prev) { - __m128i Add = _mm_slli_si128(Vec, 4); // Cycle 1: [- A B C] (already done) - Prev = _mm_shuffle_epi32(Prev, BroadcastLastXMM); // Cycle 2: [P P P P] - Vec = _mm_add_epi32(Vec, Add); // Cycle 2: [A AB BC CD] - Add = _mm_slli_si128(Vec, 8); // Cycle 3: [- - A AB] - Vec = _mm_add_epi32(Vec, Prev); // Cycle 3: [PA PAB PBC PCD] - Vec = _mm_add_epi32(Vec, Add); // Cycle 4: [PA PAB PABC PABCD] - - _write_avx(out, Vec); - return Vec; -} - -#ifndef _MSC_VER -static __m128i High16To32 = {0xFFFF0B0AFFFF0908, 0xFFFF0F0EFFFF0D0C}; -#else -static __m128i High16To32 = {8, 9, -1, -1, 10, 11, -1, -1, - 12, 13, -1, -1, 14, 15, -1, -1}; -#endif - -static inline __m128i _write_16bit_avx_d1(uint32_t *out, __m128i Vec, __m128i Prev) { - // vec == [A B C D E F G H] (16 bit values) - __m128i Add = _mm_slli_si128(Vec, 2); // [- A B C D E F G] - Prev = _mm_shuffle_epi32(Prev, BroadcastLastXMM); // [P P P P] (32-bit) - Vec = _mm_add_epi32(Vec, Add); // [A AB BC CD DE FG GH] - Add = _mm_slli_si128(Vec, 4); // [- - A AB BC CD DE EF] - Vec = _mm_add_epi32(Vec, Add); // [A AB ABC ABCD BCDE CDEF DEFG EFGH] - __m128i V1 = _mm_cvtepu16_epi32(Vec); // [A AB ABC ABCD] (32-bit) - V1 = _mm_add_epi32(V1, Prev); // [PA PAB PABC PABCD] (32-bit) - __m128i V2 = - _mm_shuffle_epi8(Vec, High16To32); // [BCDE CDEF DEFG EFGH] (32-bit) - V2 = _mm_add_epi32(V1, V2); // [PABCDE PABCDEF PABCDEFG PABCDEFGH] (32-bit) - _write_avx(out, V1); - _write_avx(out + 4, V2); - return V2; -} - -static inline uint32_t _decode_data(const uint8_t **dataPtrPtr, uint8_t code) { - const uint8_t *dataPtr = *dataPtrPtr; - uint32_t val; - - if (code == 0) { // 1 byte - val = (uint32_t) * dataPtr; - dataPtr += 1; - } else if (code == 1) { // 2 bytes - val = (uint32_t) * (uint16_t *) dataPtr; - dataPtr += 2; - } else if (code == 2) { // 3 bytes - val = (uint32_t) * (uint16_t *) dataPtr; - val |= *(dataPtr + 2) << 16; - dataPtr += 3; - } else { // code == 3 - val = *(uint32_t *) dataPtr; // 4 bytes - dataPtr += 4; - } - - *dataPtrPtr = dataPtr; - return val; -} - -const uint8_t *svb_decode_scalar_d1_init(uint32_t *outPtr, const uint8_t *keyPtr, - const uint8_t *dataPtr, uint32_t count, - uint32_t prev) { - if (count == 0) - return dataPtr; // no reads or writes if no data - - uint8_t shift = 0; - uint32_t key = *keyPtr++; - - for (uint32_t c = 0; c < count; c++) { - if (shift == 8) { - shift = 0; - key = *keyPtr++; - } - uint32_t val = _decode_data(&dataPtr, (key >> shift) & 0x3); - val += prev; - *outPtr++ = val; - prev = val; - shift += 2; - } - - return dataPtr; // pointer to first unused byte after end -} - -const uint8_t *svb_decode_avx_d1_init(uint32_t *out, const uint8_t *__restrict__ keyPtr, - const uint8_t *__restrict__ dataPtr, uint64_t count, uint32_t prev) { - uint64_t keybytes = count / 4; // number of key bytes - if (keybytes >= 8) { - __m128i Prev = _mm_set1_epi32(prev); - __m128i Data; - - int64_t Offset = -(int64_t) keybytes / 8 + 1; - - const uint64_t *keyPtr64 = (const uint64_t *) keyPtr - Offset; - uint64_t nextkeys = keyPtr64[Offset]; - for (; Offset != 0; ++Offset) { - uint64_t keys = nextkeys; - nextkeys = keyPtr64[Offset + 1]; - // faster 16-bit delta since we only have 8-bit values - if (!keys) { // 32 1-byte ints in a row - - Data = _mm_cvtepu8_epi16(_mm_lddqu_si128((__m128i *) (dataPtr))); - Prev = _write_16bit_avx_d1(out, Data, Prev); - Data = _mm_cvtepu8_epi16( - _mm_lddqu_si128((__m128i *) (dataPtr + 8))); - Prev = _write_16bit_avx_d1(out + 8, Data, Prev); - Data = _mm_cvtepu8_epi16( - _mm_lddqu_si128((__m128i *) (dataPtr + 16))); - Prev = _write_16bit_avx_d1(out + 16, Data, Prev); - Data = _mm_cvtepu8_epi16( - _mm_lddqu_si128((__m128i *) (dataPtr + 24))); - Prev = _write_16bit_avx_d1(out + 24, Data, Prev); - out += 32; - dataPtr += 32; - continue; - } - - Data = _decode_avx(keys & 0x00FF, &dataPtr); - Prev = _write_avx_d1(out, Data, Prev); - Data = _decode_avx((keys & 0xFF00) >> 8, &dataPtr); - Prev = _write_avx_d1(out + 4, Data, Prev); - - keys >>= 16; - Data = _decode_avx((keys & 0x00FF), &dataPtr); - Prev = _write_avx_d1(out + 8, Data, Prev); - Data = _decode_avx((keys & 0xFF00) >> 8, &dataPtr); - Prev = _write_avx_d1(out + 12, Data, Prev); - - keys >>= 16; - Data = _decode_avx((keys & 0x00FF), &dataPtr); - Prev = _write_avx_d1(out + 16, Data, Prev); - Data = _decode_avx((keys & 0xFF00) >> 8, &dataPtr); - Prev = _write_avx_d1(out + 20, Data, Prev); - - keys >>= 16; - Data = _decode_avx((keys & 0x00FF), &dataPtr); - Prev = _write_avx_d1(out + 24, Data, Prev); - Data = _decode_avx((keys & 0xFF00) >> 8, &dataPtr); - Prev = _write_avx_d1(out + 28, Data, Prev); - - out += 32; - } - { - uint64_t keys = nextkeys; - // faster 16-bit delta since we only have 8-bit values - if (!keys) { // 32 1-byte ints in a row - Data = _mm_cvtepu8_epi16(_mm_lddqu_si128((__m128i *) (dataPtr))); - Prev = _write_16bit_avx_d1(out, Data, Prev); - Data = _mm_cvtepu8_epi16( - _mm_lddqu_si128((__m128i *) (dataPtr + 8))); - Prev = _write_16bit_avx_d1(out + 8, Data, Prev); - Data = _mm_cvtepu8_epi16( - _mm_lddqu_si128((__m128i *) (dataPtr + 16))); - Prev = _write_16bit_avx_d1(out + 16, Data, Prev); - Data = _mm_cvtepu8_epi16( - _mm_loadl_epi64((__m128i *) (dataPtr + 24))); - Prev = _write_16bit_avx_d1(out + 24, Data, Prev); - out += 32; - dataPtr += 32; - - } else { - - Data = _decode_avx(keys & 0x00FF, &dataPtr); - Prev = _write_avx_d1(out, Data, Prev); - Data = _decode_avx((keys & 0xFF00) >> 8, &dataPtr); - Prev = _write_avx_d1(out + 4, Data, Prev); - - keys >>= 16; - Data = _decode_avx((keys & 0x00FF), &dataPtr); - Prev = _write_avx_d1(out + 8, Data, Prev); - Data = _decode_avx((keys & 0xFF00) >> 8, &dataPtr); - Prev = _write_avx_d1(out + 12, Data, Prev); - - keys >>= 16; - Data = _decode_avx((keys & 0x00FF), &dataPtr); - Prev = _write_avx_d1(out + 16, Data, Prev); - Data = _decode_avx((keys & 0xFF00) >> 8, &dataPtr); - Prev = _write_avx_d1(out + 20, Data, Prev); - - keys >>= 16; - Data = _decode_avx((keys & 0x00FF), &dataPtr); - Prev = _write_avx_d1(out + 24, Data, Prev); - Data = _decode_avx((keys & 0xFF00) >> 8, &dataPtr); - Prev = _write_avx_d1(out + 28, Data, Prev); - - out += 32; - } - } - prev = out[-1]; - } - uint64_t consumedkeys = keybytes - (keybytes & 7); - return svb_decode_scalar_d1_init(out, keyPtr + consumedkeys, dataPtr, - count & 31, prev); -} - -size_t streamvbyte_delta_decode(const uint8_t* in, uint32_t* out, - uint32_t count, uint32_t prev) { - uint32_t keyLen = ((count + 3) / 4); // 2-bits per key (rounded up) - const uint8_t *keyPtr = in; - const uint8_t *dataPtr = keyPtr + keyLen; // data starts at end of keys - return svb_decode_avx_d1_init(out, keyPtr, dataPtr, count, prev) - in; -} diff --git a/cpp/streamvbyte/tests/unit.c b/cpp/streamvbyte/tests/unit.c deleted file mode 100644 index f2870464f..000000000 --- a/cpp/streamvbyte/tests/unit.c +++ /dev/null @@ -1,73 +0,0 @@ -#include -#include - -#include "streamvbyte.h" -#include "streamvbytedelta.h" - -int main() { - int N = 4096; - uint32_t * datain = malloc(N * sizeof(uint32_t)); - uint8_t * compressedbuffer = malloc(2 * N * sizeof(uint32_t)); - uint32_t * recovdata = malloc(N * sizeof(uint32_t)); - - for (int length = 0; length <= N;) { - printf("length = %d \n", length); - for (uint32_t gap = 1; gap <= 387420489; gap *= 3) { - for (int k = 0; k < length; ++k) - datain[k] = gap; - size_t compsize = streamvbyte_encode(datain, length, - compressedbuffer); - size_t usedbytes = streamvbyte_decode(compressedbuffer, recovdata, - length); - if (compsize != usedbytes) { - printf( - "[streamvbyte_decode] code is buggy gap = %d, size mismatch %d %d \n", - (int) gap, (int) compsize, (int) usedbytes); - return -1; - } - for (int k = 0; k < length; ++k) { - if (recovdata[k] != datain[k]) { - printf("[streamvbyte_decode] code is buggy gap = %d\n", - (int) gap); - return -1; - } - } - } - - printf("Delta \n"); - for (size_t gap = 1; gap <= 531441; gap *= 3) { - for (int k = 0; k < length; ++k) - datain[k] = gap * k; - size_t compsize = streamvbyte_delta_encode(datain, length, - compressedbuffer, 0); - size_t usedbytes = streamvbyte_delta_decode(compressedbuffer, - recovdata, length, 0); - if (compsize != usedbytes) { - printf( - "[streamvbyte_delta_decode] code is buggy gap = %d, size mismatch %d %d \n", - (int) gap, (int) compsize, (int) usedbytes); - return -1; - } - for (int k = 0; k < length; ++k) { - if (recovdata[k] != datain[k]) { - printf( - "[streamvbyte_delta_decode] code is buggy gap = %d\n", - (int) gap); - return -1; - } - } - - } - - if (length < 128) - ++length; - else { - length *= 2; - } - } - free(datain); - free(compressedbuffer); - free(recovdata); - printf("Code looks good.\n"); - return 0; -} diff --git a/src/compression/mod.rs b/src/compression/mod.rs index f331ce60b..85b8e4581 100644 --- a/src/compression/mod.rs +++ b/src/compression/mod.rs @@ -3,41 +3,138 @@ mod stream; -pub use self::stream::CompressedIntStream; - pub const COMPRESSION_BLOCK_SIZE: usize = 128; +const COMPRESSED_BLOCK_MAX_SIZE: usize = COMPRESSION_BLOCK_SIZE * 4 + 1; + +pub use self::stream::CompressedIntStream; +use std::cmp; + + +use bitpacking::BitPacker; + +#[cfg(not(feature = "simdcompression"))] +pub use bitpacking::ScalarBitPacker as BitPackerImpl; +#[cfg(not(feature = "simdcompression"))] +const MINI_BLOCK: usize = 4; + +#[cfg(feature = "simdcompression")] +pub use bitpacking::SSE3BitPacker as BitPackerImpl; +#[cfg(feature = "simdcompression")] +const MINI_BLOCK: usize = 1; /// Returns the size in bytes of a compressed block, given `num_bits`. pub fn compressed_block_size(num_bits: u8) -> usize { - 1 + (num_bits as usize) * 16 + 1 + (num_bits as usize) * BitPackerImpl::BLOCK_LEN / 8 } -#[cfg(not(feature = "simdcompression"))] -mod pack { - mod compression_pack_nosimd; - pub use self::compression_pack_nosimd::{BlockDecoder, BlockEncoder}; +pub struct BlockEncoder { + pub output: [u8; COMPRESSED_BLOCK_MAX_SIZE], + pub output_len: usize, } -#[cfg(feature = "simdcompression")] -mod pack { - mod compression_pack_simd; - pub use self::compression_pack_simd::{BlockDecoder, BlockEncoder}; +impl BlockEncoder { + pub fn new() -> BlockEncoder { + BlockEncoder { + output: [0u8; COMPRESSED_BLOCK_MAX_SIZE], + output_len: 0, + } + } + + pub fn compress_block_sorted(&mut self, vals: &[u32], offset: u32) -> &[u8] { + assert_eq!(vals.len(), COMPRESSION_BLOCK_SIZE); + let mut num_bits = 0; + let mut offsets = [offset; MINI_BLOCK]; + for i in 1..MINI_BLOCK { + offsets[i] = vals[(i * BitPackerImpl::BLOCK_LEN) - 1]; + } + for i in 0..MINI_BLOCK { + let block = &vals[i * BitPackerImpl::BLOCK_LEN.. (i + 1)*BitPackerImpl::BLOCK_LEN]; + num_bits = cmp::max(BitPackerImpl::num_bits_sorted(offsets[i], block), num_bits); + } + self.output[0] = num_bits; + let compressed_chunk_len = (num_bits as usize) * BitPackerImpl::BLOCK_LEN / 8; + let mut written_size = 1; + for i in 0..MINI_BLOCK { + let block = &vals[i * BitPackerImpl::BLOCK_LEN.. (i + 1)*BitPackerImpl::BLOCK_LEN]; + BitPackerImpl::compress_sorted(offsets[i], block, &mut self.output[written_size..], num_bits); + written_size += compressed_chunk_len; + } + &self.output[..written_size] + } + + pub fn compress_block_unsorted(&mut self, vals: &[u32]) -> &[u8] { + assert_eq!(vals.len(), COMPRESSION_BLOCK_SIZE); + let num_bits = vals.chunks(BitPackerImpl::BLOCK_LEN) + .map(|chunk| BitPackerImpl::num_bits(chunk)) + .max() + .unwrap_or(0u8); + self.output[0] = num_bits; + let mut written_size = 1; + let compressed_chunk_len = (num_bits as usize) * BitPackerImpl::BLOCK_LEN / 8; + for chunk in vals.chunks(BitPackerImpl::BLOCK_LEN) { + BitPackerImpl::compress(chunk, &mut self.output[written_size..], num_bits); + written_size += compressed_chunk_len; + } + &self.output[..written_size] + } } -pub use self::pack::{BlockDecoder, BlockEncoder}; -#[cfg(any(not(feature = "simdcompression"), target_env = "msvc"))] -mod vint { - mod compression_vint_nosimd; - pub(crate) use self::compression_vint_nosimd::*; +pub struct BlockDecoder { + pub output: [u32; COMPRESSED_BLOCK_MAX_SIZE], + pub output_len: usize, } -#[cfg(all(feature = "simdcompression", not(target_env = "msvc")))] -mod vint { - mod compression_vint_simd; - pub(crate) use self::compression_vint_simd::*; +impl BlockDecoder { + pub fn new() -> BlockDecoder { + BlockDecoder::with_val(0u32) + } + + pub fn with_val(val: u32) -> BlockDecoder { + BlockDecoder { + output: [val; COMPRESSED_BLOCK_MAX_SIZE], + output_len: 0, + } + } + + pub fn uncompress_block_sorted(&mut self, compressed_data: &[u8], mut offset: u32) -> usize { + let num_bits = compressed_data[0]; + let mut read_size: usize = 1; + let chunk_size: usize = (num_bits as usize) * BitPackerImpl::BLOCK_LEN / 8; + for i in 0..MINI_BLOCK { + BitPackerImpl::decompress_sorted(offset, &compressed_data[read_size..], &mut self.output[i*BitPackerImpl::BLOCK_LEN..], num_bits); + offset = self.output[(i + 1)*BitPackerImpl::BLOCK_LEN - 1]; + read_size += chunk_size; + } + self.output_len = COMPRESSION_BLOCK_SIZE; + read_size + } + + pub fn uncompress_block_unsorted<'a>(&mut self, compressed_data: &'a [u8]) -> usize { + let num_bits = compressed_data[0]; + let mut read_size: usize = 1; + let chunk_size: usize = (num_bits as usize) * BitPackerImpl::BLOCK_LEN / 8; + for i in 0..MINI_BLOCK { + BitPackerImpl::decompress(&compressed_data[read_size..], &mut self.output[i*BitPackerImpl::BLOCK_LEN..], num_bits); + read_size += chunk_size; + } + self.output_len = COMPRESSION_BLOCK_SIZE; + read_size + } + + #[inline] + pub fn output_array(&self) -> &[u32] { + &self.output[..self.output_len] + } + + #[inline] + pub fn output(&self, idx: usize) -> u32 { + self.output[idx] + } } +mod vint; + pub trait VIntEncoder { /// Compresses an array of `u32` integers, /// using [delta-encoding](https://en.wikipedia.org/wiki/Delta_encoding) diff --git a/src/compression/pack/compression_pack_nosimd.rs b/src/compression/pack/compression_pack_nosimd.rs deleted file mode 100644 index 52421a514..000000000 --- a/src/compression/pack/compression_pack_nosimd.rs +++ /dev/null @@ -1,148 +0,0 @@ -use common::compute_num_bits; -use common::bitpacker::{BitPacker, BitUnpacker}; -use common::CountingWriter; -use std::cmp; -use std::io::Write; -use super::super::{compressed_block_size, COMPRESSION_BLOCK_SIZE}; - -const COMPRESSED_BLOCK_MAX_SIZE: usize = COMPRESSION_BLOCK_SIZE * 4 + 1; - -pub fn compress_sorted(vals: &mut [u32], output: &mut [u8], offset: u32) -> usize { - let mut max_delta = 0; - { - let mut local_offset = offset; - for i in 0..COMPRESSION_BLOCK_SIZE { - let val = vals[i]; - let delta = val - local_offset; - max_delta = cmp::max(max_delta, delta); - vals[i] = delta; - local_offset = val; - } - } - let mut counting_writer = CountingWriter::wrap(output); - let num_bits = compute_num_bits(max_delta as u64); - counting_writer.write_all(&[num_bits]).unwrap(); - - let mut bit_packer = BitPacker::new(); - for val in vals { - bit_packer - .write(*val as u64, num_bits, &mut counting_writer) - .unwrap(); - } - let compressed_size = counting_writer.written_bytes(); - assert_eq!(compressed_size, compressed_block_size(num_bits)); - compressed_size -} - -pub struct BlockEncoder { - pub output: [u8; COMPRESSED_BLOCK_MAX_SIZE], - pub output_len: usize, - input_buffer: [u32; COMPRESSION_BLOCK_SIZE], -} - -impl BlockEncoder { - pub fn new() -> BlockEncoder { - BlockEncoder { - output: [0u8; COMPRESSED_BLOCK_MAX_SIZE], - output_len: 0, - input_buffer: [0u32; COMPRESSION_BLOCK_SIZE], - } - } - - pub fn compress_block_sorted(&mut self, vals: &[u32], offset: u32) -> &[u8] { - self.input_buffer.clone_from_slice(vals); - let compressed_size = compress_sorted(&mut self.input_buffer, &mut self.output, offset); - &self.output[..compressed_size] - } - - pub fn compress_block_unsorted(&mut self, vals: &[u32]) -> &[u8] { - let compressed_size = { - let output: &mut [u8] = &mut self.output; - let max = vals.iter() - .cloned() - .max() - .expect("compress unsorted called with an empty array"); - let num_bits = compute_num_bits(max as u64); - let mut counting_writer = CountingWriter::wrap(output); - counting_writer.write_all(&[num_bits]).unwrap(); - let mut bit_packer = BitPacker::new(); - for val in vals { - bit_packer - .write(*val as u64, num_bits, &mut counting_writer) - .unwrap(); - } - for _ in vals.len()..COMPRESSION_BLOCK_SIZE { - bit_packer - .write(vals[0] as u64, num_bits, &mut counting_writer) - .unwrap(); - } - bit_packer.flush(&mut counting_writer).expect( - "Flushing the bitpacking \ - in an in RAM buffer should never fail", - ); - // we avoid writing "closing", because we - // do not want 7 bytes of padding here. - counting_writer.written_bytes() - }; - &self.output[..compressed_size] - } -} - -pub struct BlockDecoder { - pub output: [u32; COMPRESSED_BLOCK_MAX_SIZE], - pub output_len: usize, -} - -impl BlockDecoder { - pub fn new() -> BlockDecoder { - BlockDecoder::with_val(0u32) - } - - pub fn with_val(val: u32) -> BlockDecoder { - BlockDecoder { - output: [val; COMPRESSED_BLOCK_MAX_SIZE], - output_len: 0, - } - } - - pub fn uncompress_block_sorted<'a>( - &mut self, - compressed_data: &'a [u8], - mut offset: u32, - ) -> usize { - let consumed_size = { - let num_bits = compressed_data[0]; - let bit_unpacker = BitUnpacker::new(&compressed_data[1..], num_bits); - for i in 0..COMPRESSION_BLOCK_SIZE { - let delta = bit_unpacker.get(i); - let val = offset + delta as u32; - self.output[i] = val; - offset = val; - } - compressed_block_size(num_bits) - }; - self.output_len = COMPRESSION_BLOCK_SIZE; - consumed_size - } - - pub fn uncompress_block_unsorted<'a>(&mut self, compressed_data: &'a [u8]) -> usize { - let num_bits = compressed_data[0]; - let bit_unpacker = BitUnpacker::new(&compressed_data[1..], num_bits); - for i in 0..COMPRESSION_BLOCK_SIZE { - self.output[i] = bit_unpacker.get(i) as u32; - } - let consumed_size = 1 + (num_bits as usize * COMPRESSION_BLOCK_SIZE + 7) / 8; - self.output_len = COMPRESSION_BLOCK_SIZE; - consumed_size - } - - #[inline] - pub fn output_array(&self) -> &[u32] { - &self.output[..self.output_len] - } - - #[inline] - pub fn output(&self, idx: usize) -> u32 { - self.output[idx] - } -} diff --git a/src/compression/pack/compression_pack_simd.rs b/src/compression/pack/compression_pack_simd.rs deleted file mode 100644 index f7374294e..000000000 --- a/src/compression/pack/compression_pack_simd.rs +++ /dev/null @@ -1,116 +0,0 @@ -use compression::COMPRESSION_BLOCK_SIZE; - -const COMPRESSED_BLOCK_MAX_SIZE: usize = COMPRESSION_BLOCK_SIZE * 4 + 1; - -mod simdcomp { - use libc::size_t; - - extern "C" { - pub fn compress_sorted(data: *const u32, output: *mut u8, offset: u32) -> size_t; - - pub fn uncompress_sorted( - compressed_data: *const u8, - output: *mut u32, - offset: u32, - ) -> size_t; - - pub fn compress_unsorted(data: *const u32, output: *mut u8) -> size_t; - - pub fn uncompress_unsorted(compressed_data: *const u8, output: *mut u32) -> size_t; - } -} - -fn compress_sorted(vals: &[u32], output: &mut [u8], offset: u32) -> usize { - unsafe { simdcomp::compress_sorted(vals.as_ptr(), output.as_mut_ptr(), offset) } -} - -fn uncompress_sorted(compressed_data: &[u8], output: &mut [u32], offset: u32) -> usize { - unsafe { simdcomp::uncompress_sorted(compressed_data.as_ptr(), output.as_mut_ptr(), offset) } -} - -fn compress_unsorted(vals: &[u32], output: &mut [u8]) -> usize { - unsafe { simdcomp::compress_unsorted(vals.as_ptr(), output.as_mut_ptr()) } -} - -fn uncompress_unsorted(compressed_data: &[u8], output: &mut [u32]) -> usize { - unsafe { simdcomp::uncompress_unsorted(compressed_data.as_ptr(), output.as_mut_ptr()) } -} - -pub struct BlockEncoder { - pub output: [u8; COMPRESSED_BLOCK_MAX_SIZE], - pub output_len: usize, -} - -impl BlockEncoder { - pub fn new() -> BlockEncoder { - BlockEncoder { - output: [0u8; COMPRESSED_BLOCK_MAX_SIZE], - output_len: 0, - } - } - - pub fn compress_block_sorted(&mut self, vals: &[u32], offset: u32) -> &[u8] { - let compressed_size = compress_sorted(vals, &mut self.output, offset); - &self.output[..compressed_size] - } - - pub fn compress_block_unsorted(&mut self, vals: &[u32]) -> &[u8] { - let compressed_size = compress_unsorted(vals, &mut self.output); - &self.output[..compressed_size] - } -} - -pub struct BlockDecoder { - pub output: [u32; COMPRESSED_BLOCK_MAX_SIZE], - pub output_len: usize, -} - -impl BlockDecoder { - pub fn new() -> BlockDecoder { - BlockDecoder::with_val(0u32) - } - - pub fn with_val(val: u32) -> BlockDecoder { - BlockDecoder { - output: [val; COMPRESSED_BLOCK_MAX_SIZE], - output_len: 0, - } - } - - pub fn uncompress_block_sorted(&mut self, compressed_data: &[u8], offset: u32) -> usize { - let consumed_size = uncompress_sorted(compressed_data, &mut self.output, offset); - self.output_len = COMPRESSION_BLOCK_SIZE; - consumed_size - } - - pub fn uncompress_block_unsorted<'a>(&mut self, compressed_data: &'a [u8]) -> usize { - let consumed_size = uncompress_unsorted(compressed_data, &mut self.output); - self.output_len = COMPRESSION_BLOCK_SIZE; - consumed_size - } - - #[inline] - pub fn output_array(&self) -> &[u32] { - &self.output[..self.output_len] - } - - #[inline] - pub fn output(&self, idx: usize) -> u32 { - self.output[idx] - } -} - -#[cfg(test)] -mod tests { - - use super::BlockEncoder; - - #[test] - fn test_all_docs_compression_len() { - let data: Vec = (0u32..128u32).collect(); - let mut encoder = BlockEncoder::new(); - let compressed = encoder.compress_block_sorted(&data, 0u32); - assert_eq!(compressed.len(), 17); - } - -} diff --git a/src/compression/stream.rs b/src/compression/stream.rs index dcfef60c6..5c7b53391 100644 --- a/src/compression/stream.rs +++ b/src/compression/stream.rs @@ -58,6 +58,8 @@ impl CompressedIntStream { /// /// If a full block is skipped, calling /// `.skip(...)` will avoid decompressing it. + /// + /// May panic if the end of the stream is reached. pub fn skip(&mut self, mut skip_len: usize) { let available = COMPRESSION_BLOCK_SIZE - self.inner_offset; if available >= skip_len { @@ -91,7 +93,7 @@ pub mod tests { fn create_stream_buffer() -> ReadOnlySource { let mut buffer: Vec = vec![]; let mut encoder = BlockEncoder::new(); - let vals: Vec = (0u32..1_025u32).collect(); + let vals: Vec = (0u32..1152u32).collect(); for chunk in vals.chunks(COMPRESSION_BLOCK_SIZE) { let compressed_block = encoder.compress_block_unsorted(chunk); let num_bits = compressed_block[0]; diff --git a/src/compression/vint/compression_vint_nosimd.rs b/src/compression/vint.rs similarity index 100% rename from src/compression/vint/compression_vint_nosimd.rs rename to src/compression/vint.rs diff --git a/src/compression/vint/compression_vint_simd.rs b/src/compression/vint/compression_vint_simd.rs deleted file mode 100644 index e178d5f60..000000000 --- a/src/compression/vint/compression_vint_simd.rs +++ /dev/null @@ -1,72 +0,0 @@ -mod streamvbyte { - - use libc::size_t; - - extern "C" { - pub fn streamvbyte_delta_encode( - data: *const u32, - num_els: u32, - output: *mut u8, - offset: u32, - ) -> size_t; - - pub fn streamvbyte_delta_decode( - compressed_data: *const u8, - output: *mut u32, - num_els: u32, - offset: u32, - ) -> size_t; - - pub fn streamvbyte_encode(data: *const u32, num_els: u32, output: *mut u8) -> size_t; - - pub fn streamvbyte_decode( - compressed_data: *const u8, - output: *mut u32, - num_els: usize, - ) -> size_t; - } -} - -#[inline(always)] -pub(crate) fn compress_sorted<'a>(input: &[u32], output: &'a mut [u8], offset: u32) -> &'a [u8] { - let compress_length = unsafe { - streamvbyte::streamvbyte_delta_encode( - input.as_ptr(), - input.len() as u32, - output.as_mut_ptr(), - offset, - ) - }; - &output[..compress_length] -} - -#[inline(always)] -pub(crate) fn compress_unsorted<'a>(input: &[u32], output: &'a mut [u8]) -> &'a [u8] { - let compress_length = unsafe { - streamvbyte::streamvbyte_encode(input.as_ptr(), input.len() as u32, output.as_mut_ptr()) - }; - &output[..compress_length] -} - -#[inline(always)] -pub(crate) fn uncompress_sorted<'a>( - compressed_data: &'a [u8], - output: &mut [u32], - offset: u32, -) -> usize { - unsafe { - streamvbyte::streamvbyte_delta_decode( - compressed_data.as_ptr(), - output.as_mut_ptr(), - output.len() as u32, - offset, - ) - } -} - -#[inline(always)] -pub(crate) fn uncompress_unsorted<'a>(compressed_data: &'a [u8], output: &mut [u32]) -> usize { - unsafe { - streamvbyte::streamvbyte_decode(compressed_data.as_ptr(), output.as_mut_ptr(), output.len()) - } -} diff --git a/src/lib.rs b/src/lib.rs index 077dbb871..3e1694e37 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -151,6 +151,7 @@ extern crate tempdir; extern crate tempfile; extern crate time; extern crate uuid; +extern crate bitpacking; #[cfg(test)] #[macro_use] @@ -159,9 +160,6 @@ extern crate matches; #[cfg(test)] extern crate env_logger; -#[cfg(feature = "simdcompression")] -extern crate libc; - #[cfg(windows)] extern crate winapi; From a26b0ff4a2f3c5f3139c8067ffa20da41647c02d Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Mon, 19 Mar 2018 11:51:41 +0900 Subject: [PATCH 14/24] Removed exclude cpp from travis configuration --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6a2712089..683dc7c26 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,5 +34,5 @@ script: - cargo run --example simple_search - cargo doc after_success: - - cargo coveralls --exclude-pattern cpp/,src/functional_test.rs + - cargo coveralls --exclude-pattern src/functional_test.rs - cargo doc-upload From 88303d4833297b320cd2c6f83e69735f5c872492 Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Mon, 19 Mar 2018 11:53:15 +0900 Subject: [PATCH 15/24] Removed script directory --- script/build-doc.sh | 10 ---------- script/profile.sh | 5 ----- 2 files changed, 15 deletions(-) delete mode 100755 script/build-doc.sh delete mode 100644 script/profile.sh diff --git a/script/build-doc.sh b/script/build-doc.sh deleted file mode 100755 index 7c36a9919..000000000 --- a/script/build-doc.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -DEST=target/doc/tantivy/docs/ -mkdir -p $DEST - -for f in $(ls docs/*.md) -do - rustdoc $f -o $DEST --markdown-css ../../rustdoc.css --markdown-css style.css -done - -cp docs/*.css $DEST \ No newline at end of file diff --git a/script/profile.sh b/script/profile.sh deleted file mode 100644 index d5ea02407..000000000 --- a/script/profile.sh +++ /dev/null @@ -1,5 +0,0 @@ -#/bin/bash -valgrind --tool=cachegrind target/release/tantivy-bench -i /data/wiki-index -q ./queries.txt -n 3 -valgrind --tool=callgrind target/release/tantivy-bench -i /data/wiki-index -q ./queries.txt -n 3 - - From 410187dd2469f7e39dcef29d3f2911d4330ede8c Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Mon, 19 Mar 2018 11:54:10 +0900 Subject: [PATCH 16/24] Removed .vimrc --- .vimrc | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 .vimrc diff --git a/.vimrc b/.vimrc deleted file mode 100644 index 29eb2867c..000000000 --- a/.vimrc +++ /dev/null @@ -1,13 +0,0 @@ -set wildignore+=*/examples/* - -set tabstop=2 -set shiftwidth=2 -set softtabstop=2 -set expandtab -set nosmarttab - -set textwidth=100 - -autocmd BufRead *.rs :setlocal tags=./rusty-tags.vi;/ -autocmd BufWritePost *.rs :silent! exec "!rusty-tags vi -o --quiet --start-dir=" . expand('%:p:h') . "&" | redraw! - From 75d130f1ce0da3b738536c2ddcc086b771a0e52f Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Mon, 19 Mar 2018 12:01:48 +0900 Subject: [PATCH 17/24] Edited CHANGELOG --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe8b4d947..e2338dac1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +Tantivy 0.5.2 +========================== + +- Removed C code. Tantivy is now pure Rust. + Tantivy 0.5.1 ========================== - bugfix #254 : tantivy failed if no documents in a segment contained a specific field. From 234a9024704ad37463c7da7c540a012d5a74f46c Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Mon, 19 Mar 2018 12:09:22 +0900 Subject: [PATCH 18/24] Removed cc from Cargo.toml --- Cargo.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 9776a4de1..a4acb9c6b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,9 +50,6 @@ winapi = "0.2" rand = "0.3" env_logger = "0.4" -[build-dependencies] -cc = { version="1.0.0", optional=true } - [profile.release] opt-level = 3 debug = false From 59639cd31180375cc3f0e94ffe12817435c687be Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Mon, 19 Mar 2018 12:58:42 +0900 Subject: [PATCH 19/24] In sync with master. Fixed merging --- src/core/inverted_index_reader.rs | 2 +- src/indexer/merger.rs | 22 +++++++++-------- src/postings/mod.rs | 33 ++++++++++++++++--------- src/postings/postings.rs | 4 +++ src/postings/segment_postings.rs | 12 +++------ src/query/phrase_query/phrase_scorer.rs | 25 +++++-------------- 6 files changed, 49 insertions(+), 49 deletions(-) diff --git a/src/core/inverted_index_reader.rs b/src/core/inverted_index_reader.rs index aff30704c..c6f730f19 100644 --- a/src/core/inverted_index_reader.rs +++ b/src/core/inverted_index_reader.rs @@ -58,7 +58,7 @@ impl InvertedIndexReader { TermDictionaryImpl::empty(field_type), ReadOnlySource::empty(), ReadOnlySource::empty(), - DeleteBitSet::empty(), + None, record_option, ) } diff --git a/src/indexer/merger.rs b/src/indexer/merger.rs index bbdf7c2f7..e47943c45 100644 --- a/src/indexer/merger.rs +++ b/src/indexer/merger.rs @@ -6,7 +6,6 @@ use core::SerializableSegment; use indexer::SegmentSerializer; use postings::InvertedIndexSerializer; use itertools::Itertools; -use postings::Postings; use docset::DocSet; use fastfield::DeleteBitSet; use schema::{Field, Schema}; @@ -18,6 +17,7 @@ use std::cmp::{max, min}; use termdict::TermDictionary; use termdict::TermStreamer; use postings::DeleteSet; +use postings::Postings; pub struct IndexMerger { schema: Schema, @@ -206,6 +206,8 @@ impl IndexMerger { } fn write_postings(&self, serializer: &mut InvertedIndexSerializer) -> Result<()> { + + let mut positions_buffer: Vec = Vec::with_capacity(1_000); let mut delta_computer = DeltaComputer::new(); let mut indexed_fields = vec![]; @@ -314,15 +316,15 @@ impl IndexMerger { { // we make sure to only write the term iff // there is at least one document. - unreachable!(); -// let positions: &[u32] = segment_postings.positions(); -// let term_freq = segment_postings.term_freq(); -// let delta_positions = delta_computer.compute_delta(positions); -// field_serializer.write_doc( -// remapped_doc_id, -// term_freq, -// delta_positions, -// )?; + let term_freq = segment_postings.term_freq(); + segment_postings.positions(&mut positions_buffer); + + let delta_positions = delta_computer.compute_delta(&positions_buffer); + field_serializer.write_doc( + remapped_doc_id, + term_freq, + delta_positions, + )?; } if !segment_postings.advance() { break; diff --git a/src/postings/mod.rs b/src/postings/mod.rs index 7734d1d6e..1c30e00a2 100644 --- a/src/postings/mod.rs +++ b/src/postings/mod.rs @@ -103,15 +103,18 @@ pub mod tests { let inverted_index = searcher.segment_reader(0u32).inverted_index(title); let term = Term::from_field_text(title, "abc"); + let mut positions = Vec::new(); { let mut postings = inverted_index .read_postings(&term, IndexRecordOption::WithFreqsAndPositions) .unwrap(); postings.advance(); - assert_eq!(&[0, 1, 2], postings.positions()); + postings.positions(&mut positions); + assert_eq!(&[0, 1, 2], &positions[..]); postings.advance(); - assert_eq!(&[0, 5], postings.positions()); + postings.positions(&mut positions); + assert_eq!(&[0, 5], &positions[..]); } { let mut postings = inverted_index @@ -119,7 +122,8 @@ pub mod tests { .unwrap(); postings.advance(); postings.advance(); - assert_eq!(&[0, 5], postings.positions()); + postings.positions(&mut positions); + assert_eq!(&[0, 5], &positions[..]); } { @@ -128,7 +132,8 @@ pub mod tests { .unwrap(); assert_eq!(postings.skip_next(1), SkipResult::Reached); assert_eq!(postings.doc(), 1); - assert_eq!(&[0, 5], postings.positions()); + postings.positions(&mut positions); + assert_eq!(&[0, 5], &positions[..]); } { let mut postings = inverted_index @@ -136,7 +141,8 @@ pub mod tests { .unwrap(); assert_eq!(postings.skip_next(1002), SkipResult::Reached); assert_eq!(postings.doc(), 1002); - assert_eq!(&[0, 5], postings.positions()); + postings.positions(&mut positions); + assert_eq!(&[0, 5], &positions[..]); } { let mut postings = inverted_index @@ -145,12 +151,14 @@ pub mod tests { assert_eq!(postings.skip_next(100), SkipResult::Reached); assert_eq!(postings.skip_next(1002), SkipResult::Reached); assert_eq!(postings.doc(), 1002); - assert_eq!(&[0, 5], postings.positions()); + postings.positions(&mut positions); + assert_eq!(&[0, 5], &positions[..]); } } #[test] pub fn test_position_and_fieldnorm1() { + let mut positions = Vec::new(); let mut schema_builder = SchemaBuilder::default(); let text_field = schema_builder.add_text_field("text", TEXT); let schema = schema_builder.build(); @@ -223,15 +231,16 @@ pub mod tests { assert!(postings_a.advance()); assert_eq!(postings_a.doc(), 0); assert_eq!(postings_a.term_freq(), 6); - assert_eq!(postings_a.positions(), [0, 2, 4, 6, 7, 13]); - assert_eq!(postings_a.positions(), [0, 2, 4, 6, 7, 13]); + postings_a.positions(&mut positions); + assert_eq!(&positions[..], [0, 2, 4, 6, 7, 13]); assert!(postings_a.advance()); assert_eq!(postings_a.doc(), 1u32); assert_eq!(postings_a.term_freq(), 1); for i in 2u32..1000u32 { assert!(postings_a.advance()); assert_eq!(postings_a.term_freq(), 1); - assert_eq!(postings_a.positions(), [i]); + postings_a.positions(&mut positions); + assert_eq!(&positions[..], [i]); assert_eq!(postings_a.doc(), i); } assert!(!postings_a.advance()); @@ -246,7 +255,7 @@ pub mod tests { for i in 2u32..1000u32 { assert!(postings_e.advance()); assert_eq!(postings_e.term_freq(), i); - let positions = postings_e.positions(); + postings_e.positions(&mut positions); assert_eq!(positions.len(), i as usize); for j in 0..positions.len() { assert_eq!(positions[j], (j as u32)); @@ -260,6 +269,7 @@ pub mod tests { #[test] pub fn test_position_and_fieldnorm2() { + let mut positions: Vec = Vec::new(); let mut schema_builder = SchemaBuilder::default(); let text_field = schema_builder.add_text_field("text", TEXT); let schema = schema_builder.build(); @@ -288,7 +298,8 @@ pub mod tests { .unwrap(); assert!(postings.advance()); assert_eq!(postings.doc(), 1u32); - assert_eq!(postings.positions(), &[1u32, 4]); + postings.positions(&mut positions); + assert_eq!(&positions[..], &[1u32, 4]); } #[test] diff --git a/src/postings/postings.rs b/src/postings/postings.rs index f66c6434d..b415860d5 100644 --- a/src/postings/postings.rs +++ b/src/postings/postings.rs @@ -17,4 +17,8 @@ pub trait Postings: DocSet + 'static { /// Returns the list of positions of the term, expressed as a list of /// token ordinals. fn positions_with_offset(&mut self, offset: u32, output: &mut Vec); + + fn positions(&mut self, output: &mut Vec) { + self.positions_with_offset(0u32, output); + } } diff --git a/src/postings/segment_postings.rs b/src/postings/segment_postings.rs index 292239f56..154381fbd 100644 --- a/src/postings/segment_postings.rs +++ b/src/postings/segment_postings.rs @@ -9,13 +9,10 @@ use std::cmp; use fst::Streamer; use compression::compressed_block_size; use postings::{NoDelete, DeleteSet}; -use std::cell::UnsafeCell; use directory::{ReadOnlySource, SourceRead}; use postings::FreqReadingOption; use postings::serializer::PostingsSerializer; -const EMPTY_POSITIONS: [u32; 0] = [0u32; 0]; - struct PositionComputer { // store the amount of position int // before reading positions. @@ -41,8 +38,7 @@ impl PositionComputer { } // Positions can only be read once. - pub fn positions(&mut self, offset: u32, output: &mut [u32]) { - let term_freq = output.len(); + pub fn positions_with_offset(&mut self, offset: u32, output: &mut [u32]) { if let Some(num_skip) = self.position_to_skip { self.positions_stream.skip(num_skip); self.positions_stream.read(output); @@ -183,7 +179,7 @@ impl DocSet for SegmentPostings { // add the term freq. if self.position_computer.is_some() { let freqs_skipped = &self.block_cursor.freqs()[self.cur..]; - let sum_freq: u32 = freqs_skipped.iter().sum() + let sum_freq: u32 = freqs_skipped.iter().sum(); self.position_computer.as_mut() .unwrap() .add_skip(sum_freq as usize); @@ -319,10 +315,10 @@ impl Postings for SegmentPostings { } unsafe { output.set_len(term_freq); - self.position_computer.as_mut().unwrap().positions(offset, &mut output[..]) + self.position_computer.as_mut().unwrap().positions_with_offset(offset, &mut output[..]) } } else { - unimplemented!("You may not read positions twice!"); + output.clear(); } } } diff --git a/src/query/phrase_query/phrase_scorer.rs b/src/query/phrase_query/phrase_scorer.rs index f77c63d68..f31b4238e 100644 --- a/src/query/phrase_query/phrase_scorer.rs +++ b/src/query/phrase_query/phrase_scorer.rs @@ -2,7 +2,6 @@ use DocId; use docset::{DocSet, SkipResult}; use postings::Postings; use query::{Intersection, Scorer}; -use std::mem; struct PostingsWithOffset { @@ -48,7 +47,7 @@ pub struct PhraseScorer { right: Vec } -fn intersection_arr(left: &mut [u32], right: &[u32]) -> usize { +fn intersection_count(left: &[u32], right: &[u32]) -> usize { let mut left_i = 0; let mut right_i = 0; let mut count = 0; @@ -58,7 +57,6 @@ fn intersection_arr(left: &mut [u32], right: &[u32]) -> usize { } else if right[right_i] < left[left_i] { right_i += 1; } else { - left[count] = left[left_i]; count+=1; left_i += 1; right_i += 1; @@ -95,7 +93,7 @@ impl PhraseScorer { { self.intersection_docset.docset_mut_specialized(i).positions(&mut self.right); } - intersection_len = intersection_arr(&mut self.left[..intersection_len], &self.right[..]); + intersection_len = intersection_count(&mut self.left[..intersection_len], &self.right[..]); if intersection_len == 0 { return false; } @@ -152,25 +150,14 @@ mod tests { use tests; use test::Bencher; - use super::{intersection_arr, intersection_avx}; + use super::intersection_count; #[bench] fn bench_intersection(b: &mut Bencher) { - let left = tests::sample_with_seed(100_000, 0.1, 1); - let right = tests::sample_with_seed(200_000, 0.05, 2); - let mut output = vec![0u32; 200_000]; + let left = tests::sample_with_seed(10, 0.1, 1); + let right = tests::sample_with_seed(2, 0.05, 2); b.iter(|| { - intersection_arr(&left, &right, &mut output); - }); - } - - #[bench] - fn bench_intersection_avx(b: &mut Bencher) { - let left = tests::sample_with_seed(100_000, 0.1, 1); - let right = tests::sample_with_seed(200_000, 0.05, 2); - let mut output = vec![0u32; 200_000]; - b.iter(|| { - intersection_avx(&left, &right, &mut output); + intersection_count(&left, &right); }); } } \ No newline at end of file From 07a8023a3a27f094d918200c7153d767a25a9994 Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Mon, 19 Mar 2018 14:36:43 +0900 Subject: [PATCH 20/24] Added --- src/query/phrase_query/phrase_scorer.rs | 44 ++++++++++++++++++++----- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/src/query/phrase_query/phrase_scorer.rs b/src/query/phrase_query/phrase_scorer.rs index f31b4238e..d8dc47300 100644 --- a/src/query/phrase_query/phrase_scorer.rs +++ b/src/query/phrase_query/phrase_scorer.rs @@ -3,7 +3,6 @@ use docset::{DocSet, SkipResult}; use postings::Postings; use query::{Intersection, Scorer}; - struct PostingsWithOffset { offset: u32, postings: TPostings @@ -47,17 +46,43 @@ pub struct PhraseScorer { right: Vec } -fn intersection_count(left: &[u32], right: &[u32]) -> usize { + +fn intersection_count(left: &mut [u32], right: &[u32]) -> usize { let mut left_i = 0; let mut right_i = 0; let mut count = 0; while left_i < left.len() && right_i < right.len() { - if left[left_i] < right[right_i] { + let left_val = left[left_i]; + let right_val = right[right_i]; + if left_val < right_val { left_i += 1; - } else if right[right_i] < left[left_i] { + } else if right_val < left_val { right_i += 1; } else { - count+=1; + count += 1; + left_i += 1; + right_i += 1; + } + } + count +} + +fn intersection_arr(left: &mut [u32], right: &[u32]) -> usize { + let mut left_i = 0; + let mut right_i = 0; + let mut count = 0; + let left_len = left.len(); + let right_len = right.len(); + while left_i < left_len && right_i < right_len { + let left_val = left[left_i]; + let right_val = right[right_i]; + if left_val < right_val { + left_i += 1; + } else if right_val < left_val { + right_i += 1; + } else { + left[count] = left_val; + count += 1; left_i += 1; right_i += 1; } @@ -89,16 +114,19 @@ impl PhraseScorer { self.intersection_docset.docset_mut_specialized(0).positions(&mut self.left); } let mut intersection_len = self.left.len(); - for i in 1..self.num_docsets { + for i in 1..self.num_docsets - 1 { { self.intersection_docset.docset_mut_specialized(i).positions(&mut self.right); } - intersection_len = intersection_count(&mut self.left[..intersection_len], &self.right[..]); + intersection_len = intersection_arr(&mut self.left[..intersection_len], &self.right[..]); if intersection_len == 0 { return false; } } - return true; + + self.intersection_docset.docset_mut_specialized(self.num_docsets - 1).positions(&mut self.right); + intersection_len = intersection_count(&mut self.left[..intersection_len], &self.right[..]); + intersection_len > 0 } } From 63064601a7761e9e3042a1cc641446e177315876 Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Tue, 20 Mar 2018 10:04:36 +0900 Subject: [PATCH 21/24] Readded test for reading positions twice --- src/postings/mod.rs | 2 ++ src/query/phrase_query/phrase_scorer.rs | 29 +++++++++++++++++++------ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/postings/mod.rs b/src/postings/mod.rs index 1c30e00a2..e7dae4701 100644 --- a/src/postings/mod.rs +++ b/src/postings/mod.rs @@ -112,6 +112,8 @@ pub mod tests { postings.advance(); postings.positions(&mut positions); assert_eq!(&[0, 1, 2], &positions[..]); + postings.positions(&mut positions); + assert_eq!(&[0, 1, 2], &positions[..]); postings.advance(); postings.positions(&mut positions); assert_eq!(&[0, 5], &positions[..]); diff --git a/src/query/phrase_query/phrase_scorer.rs b/src/query/phrase_query/phrase_scorer.rs index d8dc47300..d841746c1 100644 --- a/src/query/phrase_query/phrase_scorer.rs +++ b/src/query/phrase_query/phrase_scorer.rs @@ -47,7 +47,8 @@ pub struct PhraseScorer { } -fn intersection_count(left: &mut [u32], right: &[u32]) -> usize { +/// Computes the length of the intersection of two sorted arrays. +fn intersection_count(left: &[u32], right: &[u32]) -> usize { let mut left_i = 0; let mut right_i = 0; let mut count = 0; @@ -67,7 +68,11 @@ fn intersection_count(left: &mut [u32], right: &[u32]) -> usize { count } -fn intersection_arr(left: &mut [u32], right: &[u32]) -> usize { +/// Intersect twos sorted arrays `left` and `right` and outputs the +/// resulting array in left. +/// +/// Returns the length of the intersection +fn intersection(left: &mut [u32], right: &[u32]) -> usize { let mut left_i = 0; let mut right_i = 0; let mut count = 0; @@ -118,7 +123,7 @@ impl PhraseScorer { { self.intersection_docset.docset_mut_specialized(i).positions(&mut self.right); } - intersection_len = intersection_arr(&mut self.left[..intersection_len], &self.right[..]); + intersection_len = intersection(&mut self.left[..intersection_len], &self.right[..]); if intersection_len == 0 { return false; } @@ -178,13 +183,23 @@ mod tests { use tests; use test::Bencher; - use super::intersection_count; + use super::{intersection_count, intersection}; #[bench] - fn bench_intersection(b: &mut Bencher) { - let left = tests::sample_with_seed(10, 0.1, 1); - let right = tests::sample_with_seed(2, 0.05, 2); + fn bench_intersection_short(b: &mut Bencher) { b.iter(|| { + let mut left = [1, 5, 10, 12]; + let right = [5, 7]; + intersection(&mut left, &right); + }); + } + + + #[bench] + fn bench_intersection_count_short(b: &mut Bencher) { + b.iter(|| { + let mut left = [1, 5, 10, 12]; + let right = [5, 7]; intersection_count(&left, &right); }); } From baaae3f4ecf2e3e17856b34ec75cc124f23c6454 Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Tue, 20 Mar 2018 11:36:22 +0900 Subject: [PATCH 22/24] Making it possible to read positions twice --- src/compression/mod.rs | 6 +- src/compression/stream.rs | 96 ++++++++++++++++--------- src/directory/read_only_source.rs | 10 +++ src/query/phrase_query/phrase_scorer.rs | 3 +- 4 files changed, 76 insertions(+), 39 deletions(-) diff --git a/src/compression/mod.rs b/src/compression/mod.rs index 85b8e4581..b593a513a 100644 --- a/src/compression/mod.rs +++ b/src/compression/mod.rs @@ -81,7 +81,7 @@ impl BlockEncoder { pub struct BlockDecoder { - pub output: [u32; COMPRESSED_BLOCK_MAX_SIZE], + pub output: [u32; COMPRESSION_BLOCK_SIZE + 1], pub output_len: usize, } @@ -91,8 +91,10 @@ impl BlockDecoder { } pub fn with_val(val: u32) -> BlockDecoder { + let mut output = [val; COMPRESSION_BLOCK_SIZE + 1]; + output[COMPRESSION_BLOCK_SIZE] = 0u32; BlockDecoder { - output: [val; COMPRESSED_BLOCK_MAX_SIZE], + output, output_len: 0, } } diff --git a/src/compression/stream.rs b/src/compression/stream.rs index 5c7b53391..8cca8a3b6 100644 --- a/src/compression/stream.rs +++ b/src/compression/stream.rs @@ -11,7 +11,12 @@ use directory::{ReadOnlySource, SourceRead}; /// decompressing blocks that are not required. pub struct CompressedIntStream { buffer: SourceRead, + block_decoder: BlockDecoder, + cached_addr: usize, // address of the currently decoded block + cached_next_addr: usize, // address following the currently decoded block + + addr: usize, // address of the block associated to the current position inner_offset: usize, } @@ -21,34 +26,47 @@ impl CompressedIntStream { CompressedIntStream { buffer: SourceRead::from(source), block_decoder: BlockDecoder::new(), - inner_offset: COMPRESSION_BLOCK_SIZE, + cached_addr: usize::max_value(), + cached_next_addr: usize::max_value(), + + addr: 0, + inner_offset: 0, } } - /// Fills a buffer with the next `output.len()` integers, - /// and advance the stream by that many els. + /// Loads the block at the given address and return the address of the + /// following block + pub fn read_block(&mut self, addr: usize) -> usize { + if self.cached_addr == addr { + // we are already on this block. + // no need to read. + self.cached_next_addr + } else { + let next_addr = addr + self.block_decoder.uncompress_block_unsorted(self.buffer.slice_from(addr)); + self.cached_addr = addr; + self.cached_next_addr = next_addr; + next_addr + } + } + + /// Fills a buffer with the next `output.len()` integers. + /// This does not consume / advance the stream. pub fn read(&mut self, output: &mut [u32]) { + let mut cursor = self.addr; + let mut inner_offset = self.inner_offset; let mut num_els: usize = output.len(); - let mut start: usize = 0; + let mut start = 0; loop { - let available = COMPRESSION_BLOCK_SIZE - self.inner_offset; - if num_els >= available { - if available > 0 { - let uncompressed_block = - &self.block_decoder.output_array()[self.inner_offset..]; - output[start..][..available].clone_from_slice(uncompressed_block); - } - num_els -= available; - start += available; - let num_consumed_bytes = self.block_decoder - .uncompress_block_unsorted(self.buffer.as_ref()); - self.buffer.advance(num_consumed_bytes); - self.inner_offset = 0; + cursor = self.read_block(cursor); + let block = &self.block_decoder.output_array()[inner_offset..]; + let block_len = block.len(); + if num_els >= block_len { + output[start..start + block_len].clone_from_slice(&block); + start += block_len; + num_els -= block_len; + inner_offset = 0; } else { - let uncompressed_block = &self.block_decoder.output_array() - [self.inner_offset..self.inner_offset + num_els]; - output[start..][..num_els].clone_from_slice(uncompressed_block); - self.inner_offset += num_els; + output[start..].clone_from_slice(&block[..num_els]); break; } } @@ -61,22 +79,19 @@ impl CompressedIntStream { /// /// May panic if the end of the stream is reached. pub fn skip(&mut self, mut skip_len: usize) { - let available = COMPRESSION_BLOCK_SIZE - self.inner_offset; - if available >= skip_len { - self.inner_offset += skip_len; - } else { - skip_len -= available; - // entirely skip decompressing some blocks. - while skip_len >= COMPRESSION_BLOCK_SIZE { - skip_len -= COMPRESSION_BLOCK_SIZE; - let num_bits: u8 = self.buffer.as_ref()[0]; + loop { + let available = COMPRESSION_BLOCK_SIZE - self.inner_offset; + if available >= skip_len { + self.inner_offset += skip_len; + break; + } else { + skip_len -= available; + // entirely skip decompressing some blocks. + let num_bits: u8 = self.buffer.get(self.addr); let block_len = compressed_block_size(num_bits); - self.buffer.advance(block_len); + self.addr += block_len; + self.inner_offset = 0; } - let num_consumed_bytes = self.block_decoder - .uncompress_block_unsorted(self.buffer.as_ref()); - self.buffer.advance(num_consumed_bytes); - self.inner_offset = skip_len; } } } @@ -115,13 +130,24 @@ pub mod tests { stream.read(&mut block[0..2]); assert_eq!(block[0], 0); assert_eq!(block[1], 1); + + // reading does not consume the stream + stream.read(&mut block[0..2]); + assert_eq!(block[0], 0); + assert_eq!(block[1], 1); + stream.skip(2); + stream.skip(5); stream.read(&mut block[0..3]); + stream.skip(3); + assert_eq!(block[0], 7); assert_eq!(block[1], 8); assert_eq!(block[2], 9); stream.skip(500); stream.read(&mut block[0..3]); + stream.skip(3); + assert_eq!(block[0], 510); assert_eq!(block[1], 511); assert_eq!(block[2], 512); diff --git a/src/directory/read_only_source.rs b/src/directory/read_only_source.rs index 161fec11a..e7e545313 100644 --- a/src/directory/read_only_source.rs +++ b/src/directory/read_only_source.rs @@ -122,6 +122,16 @@ impl SourceRead { pub fn advance(&mut self, len: usize) { self.cursor = &self.cursor[len..]; } + + pub fn slice_from(&self, start: usize) -> &[u8] { + &self.cursor[start..] + + } + + pub fn get(&self, idx: usize) -> u8 { + self.cursor[idx] + } + } impl AsRef<[u8]> for SourceRead { diff --git a/src/query/phrase_query/phrase_scorer.rs b/src/query/phrase_query/phrase_scorer.rs index d841746c1..83c7f79ac 100644 --- a/src/query/phrase_query/phrase_scorer.rs +++ b/src/query/phrase_query/phrase_scorer.rs @@ -181,7 +181,6 @@ impl Scorer for PhraseScorer { #[cfg(test)] mod tests { - use tests; use test::Bencher; use super::{intersection_count, intersection}; @@ -198,7 +197,7 @@ mod tests { #[bench] fn bench_intersection_count_short(b: &mut Bencher) { b.iter(|| { - let mut left = [1, 5, 10, 12]; + let left = [1, 5, 10, 12]; let right = [5, 7]; intersection_count(&left, &right); }); From 39b182c24bcfc54a8b3f34631f7dee5c449969a1 Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Tue, 20 Mar 2018 11:47:48 +0900 Subject: [PATCH 23/24] Simplified phrase queries. Reading several time is ok. --- src/postings/segment_postings.rs | 26 ++++++++++--------------- src/query/phrase_query/phrase_scorer.rs | 4 +++- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/src/postings/segment_postings.rs b/src/postings/segment_postings.rs index 154381fbd..0ed7ca5c5 100644 --- a/src/postings/segment_postings.rs +++ b/src/postings/segment_postings.rs @@ -19,37 +19,31 @@ struct PositionComputer { // // if none, position are already loaded in // the positions vec. - position_to_skip: Option, + position_to_skip: usize, positions_stream: CompressedIntStream, } impl PositionComputer { pub fn new(positions_stream: CompressedIntStream) -> PositionComputer { PositionComputer { - position_to_skip: None, + position_to_skip: 0, positions_stream, } } pub fn add_skip(&mut self, num_skip: usize) { - self.position_to_skip = self.position_to_skip - .map(|prev_skip| prev_skip + num_skip) - .or(Some(0)); + self.position_to_skip += num_skip; } // Positions can only be read once. pub fn positions_with_offset(&mut self, offset: u32, output: &mut [u32]) { - if let Some(num_skip) = self.position_to_skip { - self.positions_stream.skip(num_skip); - self.positions_stream.read(output); - self.position_to_skip = None; - let mut cum = offset; - for output_mut in output.iter_mut() { - cum += *output_mut; - *output_mut = cum; - } - } else { - panic!("Positions have already been read."); + self.positions_stream.skip(self.position_to_skip); + self.position_to_skip = 0; + self.positions_stream.read(output); + let mut cum = offset; + for output_mut in output.iter_mut() { + cum += *output_mut; + *output_mut = cum; } } } diff --git a/src/query/phrase_query/phrase_scorer.rs b/src/query/phrase_query/phrase_scorer.rs index 83c7f79ac..8b1ef277e 100644 --- a/src/query/phrase_query/phrase_scorer.rs +++ b/src/query/phrase_query/phrase_scorer.rs @@ -116,7 +116,9 @@ impl PhraseScorer { fn phrase_match(&mut self) -> bool { // TODO early exit when we don't care about the phrase frequency { - self.intersection_docset.docset_mut_specialized(0).positions(&mut self.left); + self.intersection_docset + .docset_mut_specialized(0) + .positions(&mut self.left); } let mut intersection_len = self.left.len(); for i in 1..self.num_docsets - 1 { From 6d47634616e9a050ef3e0bd11886f0a746a9e13a Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Tue, 20 Mar 2018 12:11:28 +0900 Subject: [PATCH 24/24] Added unit tests --- src/query/phrase_query/phrase_scorer.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/query/phrase_query/phrase_scorer.rs b/src/query/phrase_query/phrase_scorer.rs index 8b1ef277e..6709c57c3 100644 --- a/src/query/phrase_query/phrase_scorer.rs +++ b/src/query/phrase_query/phrase_scorer.rs @@ -186,6 +186,29 @@ mod tests { use test::Bencher; use super::{intersection_count, intersection}; + + fn test_intersection_sym(left: &[u32], right: &[u32], expected: &[u32]) { + test_intersection_aux(left, right, expected); + test_intersection_aux(right, left, expected); + } + + fn test_intersection_aux(left: &[u32], right: &[u32], expected: &[u32]) { + let mut left_vec = Vec::from(left); + let left_mut = &mut left_vec[..]; + assert_eq!(intersection_count(left_mut, right), expected.len()); + let count = intersection(left_mut, right); + assert_eq!(&left_mut[..count], expected); + } + + #[test] + fn test_intersection() { + test_intersection_sym(&[1], &[1], &[1]); + test_intersection_sym(&[1], &[2], &[]); + test_intersection_sym(&[], &[2], &[]); + test_intersection_sym(&[5, 7], &[1, 5, 10, 12], &[5]); + test_intersection_sym(&[1, 5, 6, 9, 10, 12], &[6, 8, 9, 12], &[6, 9, 12]); + } + #[bench] fn bench_intersection_short(b: &mut Bencher) { b.iter(|| {