diff --git a/src/datastruct/bytes_hashmap/hash_item.rs b/src/datastruct/bytes_hashmap/hash_item.rs deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/datastruct/bytes_hashmap/hash_map.rs b/src/datastruct/bytes_hashmap/hash_map.rs deleted file mode 100644 index e1b9e8b43..000000000 --- a/src/datastruct/bytes_hashmap/hash_map.rs +++ /dev/null @@ -1,69 +0,0 @@ -use super::MemoryPool; -use core::hash::SipHasher; - - -const HASH_TABLE_SIZE: usize = 1 << 20; - -#[derive(Copy)] -pub struct HashItem { - pub start: u32, - pub stop: u32, -} - -impl Default for HashItem { - fn default() -> HashItem { - HashItem { - start: 0, - stop: 0, - } - } -} - -impl HashItem { - pub fn is_empty(&self,) -> bool { - self.stop == self.start - } -} - -pub struct HashMap { - sip_hasher: SipHasher, - memory_pool: MemoryPool, - hash_table: [HashItem; HASH_TABLE_SIZE], -} - - - -impl HashMap { - - pub fn new(num_bytes: usize) { - HashMap { - sip_hasher: SipHasher::new(), - memory_pool: MemoryPool::with_capacity(num_bytes), - hash_table: [HashItem::default(); HASH_TABLE_SIZE], - } - } - - fn bucket_id(&mut self, term: &[u8]) -> usize { - self.sip_hasher.write(term); - let hash: u64 = self.finish(); - (hash as usize) % HASH_TABLE_SIZE - } - - pub fn get_or_put(&mut self, term: &[u8]) -> HashItem { - let mut bucket_id = self.bucket_id(term); - loop { - let hash_item = &self.hash_table[bucket_id]; - if hash_item.is_empty() { - return self.memory_pool.store(term) - } - else { - let current_term = self.memory_pool.get(item); - if current_term == term { - return hash_item; - } - } - bucket_id = (bucket_id + 1) % HASH_TABLE_SIZE; - } - } - -} \ No newline at end of file diff --git a/src/datastruct/bytes_hashmap/memory_pool.rs b/src/datastruct/bytes_hashmap/memory_pool.rs deleted file mode 100644 index 9105209c4..000000000 --- a/src/datastruct/bytes_hashmap/memory_pool.rs +++ /dev/null @@ -1,43 +0,0 @@ -use std::iter; -use super::HashItem; - -pub struct MemoryPool { - data: Box<[u8]>, - offset: usize, -} - - -impl MemoryPool { - pub fn with_capacity(num_bytes: usize) -> MemoryPool { - let buffer_vec: Vec = iter::repeat(0u8).take(num_bytes).collect(); - MemoryPool { - data: buffer_vec.into_boxed_slice(), - offset: 0, - } - } - - pub fn get(&self, item: HashItem) -> &[u8] { - &self.data.as_ref()[item.start .. item.stop] - } - - pub fn get_mut(&self, item: HashItem) -> &mut [u8] { - &self.data.as_mut()[item.start .. item.stop] - } - - pub fn allocate(&mut self, allocate_size: usize) -> usize { - let offset = self.offset; - self.offset += allocate_size; - offset - } - - pub fn store(&mut self, term: &[u8]) -> HashItem { - let offset = self.offset; - let term_len = term.len(); - self.data.as_mut()[offset..offset + term_len].clone_from_slice(term); - self.offset += term_len; - HashItem { - start: offset, - stop: self.offset, - } - } -} \ No newline at end of file diff --git a/src/datastruct/bytes_hashmap/mod.rs b/src/datastruct/bytes_hashmap/mod.rs deleted file mode 100644 index 18822b7e5..000000000 --- a/src/datastruct/bytes_hashmap/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -mod memory_pool; -mod hash_item; -mod hash_map; - -pub use hash_item::HashItem; -pub use hash_map::HashMap; -pub use memory_pool::MemoryPool; \ No newline at end of file diff --git a/src/indexer/index_writer.rs b/src/indexer/index_writer.rs index e15f63bbb..6c688c08a 100644 --- a/src/indexer/index_writer.rs +++ b/src/indexer/index_writer.rs @@ -30,7 +30,7 @@ pub const HEAP_SIZE_LIMIT: u32 = MARGIN_IN_BYTES * 3u32; const PIPELINE_MAX_SIZE_IN_DOCS: usize = 10_000; pub struct IndexWriter { - index: Index, + index: Index, heap_size_in_bytes_per_thread: usize, workers_join_handle: Vec>, segment_ready_sender: chan::Sender>, @@ -74,7 +74,7 @@ impl IndexWriter { let document_receiver_clone = self.document_receiver.clone(); let mut heap = Heap::with_capacity(self.heap_size_in_bytes_per_thread); - let join_handle: JoinHandle<()> = thread::spawn(move || { + let join_handle: JoinHandle<()> = thread::spawn(move || { loop { let segment = index.new_segment(); let segment_id = segment.id(); @@ -104,7 +104,7 @@ impl IndexWriter { /// num_threads tells the number of indexing worker that /// should work at the same time. pub fn open(index: &Index, - num_threads: usize, + num_threads: usize, heap_size_in_bytes_per_thread: usize) -> Result { if heap_size_in_bytes_per_thread <= HEAP_SIZE_LIMIT as usize { panic!(format!("The heap size per thread needs to be at least {}.", HEAP_SIZE_LIMIT)); @@ -262,7 +262,7 @@ impl IndexWriter { /// /// Currently it represents the number of documents that /// have been added since the creation of the index. - pub fn add_document(&mut self, doc: Document) -> io::Result { + pub fn add_document(&mut self, doc: Document) -> io::Result { self.document_sender.send(doc); self.docstamp += 1; Ok(self.docstamp) @@ -290,7 +290,7 @@ mod tests { let num_docs_containing = |s: &str| { let searcher = index.searcher(); let term_a = Term::from_field_text(text_field, s); - searcher.doc_freq(&term_a) + searcher.doc_freq(&term_a) }; { diff --git a/src/postings/postings_writer.rs b/src/postings/postings_writer.rs index a2e202db9..2a204ecda 100644 --- a/src/postings/postings_writer.rs +++ b/src/postings/postings_writer.rs @@ -70,7 +70,7 @@ impl<'a, Rec: Recorder + 'static> SpecializedPostingsWriter<'a, Rec> { term_index: HashMap::new(hashmap_size, heap), } } - + pub fn new_boxed(heap: &'a Heap) -> Box { let res = SpecializedPostingsWriter::::new(heap); Box::new(res)