mirror of
https://github.com/quickwit-oss/tantivy.git
synced 2026-10-30 15:27:12 +00:00
Removed dead code.
This commit is contained in:
@@ -1,69 +0,0 @@
|
||||
use super::MemoryPool;
|
||||
use core::hash::SipHasher;
|
||||
|
||||
|
||||
const HASH_TABLE_SIZE: usize = 1 << 20;
|
||||
|
||||
#[derive(Copy)]
|
||||
pub struct HashItem {
|
||||
pub start: u32,
|
||||
pub stop: u32,
|
||||
}
|
||||
|
||||
impl Default for HashItem {
|
||||
fn default() -> HashItem {
|
||||
HashItem {
|
||||
start: 0,
|
||||
stop: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl HashItem {
|
||||
pub fn is_empty(&self,) -> bool {
|
||||
self.stop == self.start
|
||||
}
|
||||
}
|
||||
|
||||
pub struct HashMap {
|
||||
sip_hasher: SipHasher,
|
||||
memory_pool: MemoryPool,
|
||||
hash_table: [HashItem; HASH_TABLE_SIZE],
|
||||
}
|
||||
|
||||
|
||||
|
||||
impl HashMap {
|
||||
|
||||
pub fn new(num_bytes: usize) {
|
||||
HashMap {
|
||||
sip_hasher: SipHasher::new(),
|
||||
memory_pool: MemoryPool::with_capacity(num_bytes),
|
||||
hash_table: [HashItem::default(); HASH_TABLE_SIZE],
|
||||
}
|
||||
}
|
||||
|
||||
fn bucket_id(&mut self, term: &[u8]) -> usize {
|
||||
self.sip_hasher.write(term);
|
||||
let hash: u64 = self.finish();
|
||||
(hash as usize) % HASH_TABLE_SIZE
|
||||
}
|
||||
|
||||
pub fn get_or_put(&mut self, term: &[u8]) -> HashItem {
|
||||
let mut bucket_id = self.bucket_id(term);
|
||||
loop {
|
||||
let hash_item = &self.hash_table[bucket_id];
|
||||
if hash_item.is_empty() {
|
||||
return self.memory_pool.store(term)
|
||||
}
|
||||
else {
|
||||
let current_term = self.memory_pool.get(item);
|
||||
if current_term == term {
|
||||
return hash_item;
|
||||
}
|
||||
}
|
||||
bucket_id = (bucket_id + 1) % HASH_TABLE_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
use std::iter;
|
||||
use super::HashItem;
|
||||
|
||||
pub struct MemoryPool {
|
||||
data: Box<[u8]>,
|
||||
offset: usize,
|
||||
}
|
||||
|
||||
|
||||
impl MemoryPool {
|
||||
pub fn with_capacity(num_bytes: usize) -> MemoryPool {
|
||||
let buffer_vec: Vec<u8> = iter::repeat(0u8).take(num_bytes).collect();
|
||||
MemoryPool {
|
||||
data: buffer_vec.into_boxed_slice(),
|
||||
offset: 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get(&self, item: HashItem) -> &[u8] {
|
||||
&self.data.as_ref()[item.start .. item.stop]
|
||||
}
|
||||
|
||||
pub fn get_mut(&self, item: HashItem) -> &mut [u8] {
|
||||
&self.data.as_mut()[item.start .. item.stop]
|
||||
}
|
||||
|
||||
pub fn allocate(&mut self, allocate_size: usize) -> usize {
|
||||
let offset = self.offset;
|
||||
self.offset += allocate_size;
|
||||
offset
|
||||
}
|
||||
|
||||
pub fn store(&mut self, term: &[u8]) -> HashItem {
|
||||
let offset = self.offset;
|
||||
let term_len = term.len();
|
||||
self.data.as_mut()[offset..offset + term_len].clone_from_slice(term);
|
||||
self.offset += term_len;
|
||||
HashItem {
|
||||
start: offset,
|
||||
stop: self.offset,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
mod memory_pool;
|
||||
mod hash_item;
|
||||
mod hash_map;
|
||||
|
||||
pub use hash_item::HashItem;
|
||||
pub use hash_map::HashMap;
|
||||
pub use memory_pool::MemoryPool;
|
||||
@@ -30,7 +30,7 @@ pub const HEAP_SIZE_LIMIT: u32 = MARGIN_IN_BYTES * 3u32;
|
||||
const PIPELINE_MAX_SIZE_IN_DOCS: usize = 10_000;
|
||||
|
||||
pub struct IndexWriter {
|
||||
index: Index,
|
||||
index: Index,
|
||||
heap_size_in_bytes_per_thread: usize,
|
||||
workers_join_handle: Vec<JoinHandle<()>>,
|
||||
segment_ready_sender: chan::Sender<Result<(SegmentId, usize)>>,
|
||||
@@ -74,7 +74,7 @@ impl IndexWriter {
|
||||
let document_receiver_clone = self.document_receiver.clone();
|
||||
|
||||
let mut heap = Heap::with_capacity(self.heap_size_in_bytes_per_thread);
|
||||
let join_handle: JoinHandle<()> = thread::spawn(move || {
|
||||
let join_handle: JoinHandle<()> = thread::spawn(move || {
|
||||
loop {
|
||||
let segment = index.new_segment();
|
||||
let segment_id = segment.id();
|
||||
@@ -104,7 +104,7 @@ impl IndexWriter {
|
||||
/// num_threads tells the number of indexing worker that
|
||||
/// should work at the same time.
|
||||
pub fn open(index: &Index,
|
||||
num_threads: usize,
|
||||
num_threads: usize,
|
||||
heap_size_in_bytes_per_thread: usize) -> Result<IndexWriter> {
|
||||
if heap_size_in_bytes_per_thread <= HEAP_SIZE_LIMIT as usize {
|
||||
panic!(format!("The heap size per thread needs to be at least {}.", HEAP_SIZE_LIMIT));
|
||||
@@ -262,7 +262,7 @@ impl IndexWriter {
|
||||
///
|
||||
/// Currently it represents the number of documents that
|
||||
/// have been added since the creation of the index.
|
||||
pub fn add_document(&mut self, doc: Document) -> io::Result<u64> {
|
||||
pub fn add_document(&mut self, doc: Document) -> io::Result<u64> {
|
||||
self.document_sender.send(doc);
|
||||
self.docstamp += 1;
|
||||
Ok(self.docstamp)
|
||||
@@ -290,7 +290,7 @@ mod tests {
|
||||
let num_docs_containing = |s: &str| {
|
||||
let searcher = index.searcher();
|
||||
let term_a = Term::from_field_text(text_field, s);
|
||||
searcher.doc_freq(&term_a)
|
||||
searcher.doc_freq(&term_a)
|
||||
};
|
||||
|
||||
{
|
||||
|
||||
@@ -70,7 +70,7 @@ impl<'a, Rec: Recorder + 'static> SpecializedPostingsWriter<'a, Rec> {
|
||||
term_index: HashMap::new(hashmap_size, heap),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
pub fn new_boxed(heap: &'a Heap) -> Box<PostingsWriter + 'a> {
|
||||
let res = SpecializedPostingsWriter::<Rec>::new(heap);
|
||||
Box::new(res)
|
||||
|
||||
Reference in New Issue
Block a user