rust/tantivy
1
0
Fork 0
mirror of https://github.com/quickwit-oss/tantivy.git synced 2026-06-01 16:10:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Format

Browse Source
This commit is contained in:
Paul Masurel
2017-08-28 17:42:26 +09:00
parent 8d05b8f7b2
commit f8710bd4b0
90 changed files with 2291 additions and 1795 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
src/datastruct/skip/skiplist_builder.rs
View File

@@ -39,11 +39,11 @@ impl<T: BinarySerializable> LayerBuilder<T> {
doc_id.serialize(&mut self.buffer)?;
value.serialize(&mut self.buffer)?;
Ok(if self.remaining == 0 {
self.remaining = self.period;
Some((doc_id, offset))
} else {
None
})
self.remaining = self.period;
Some((doc_id, offset))
} else {
None
})
}
}
@@ -78,8 +78,10 @@ impl<T: BinarySerializable> SkipListBuilder<T> {
loop {
skip_pointer = match skip_pointer {
Some((skip_doc_id, skip_offset)) => {
try!(self.get_skip_layer(layer_id)
.insert(skip_doc_id, &skip_offset))
try!(self.get_skip_layer(layer_id).insert(
skip_doc_id,
&skip_offset,
))
}
None => {
return Ok(());

40
src/datastruct/stacker/hashmap.rs
View File

@@ -68,9 +68,14 @@ pub(crate) fn split_memory(per_thread_memory_budget: usize) -> (usize, usize) {
};
let table_num_bits: usize = (1..)
.into_iter()
.take_while(|num_bits: &usize| compute_table_size(*num_bits) < table_size_limit)
.take_while(|num_bits: &usize| {
compute_table_size(*num_bits) < table_size_limit
})
.last()
.expect(&format!("Per thread memory is too small: {}", per_thread_memory_budget));
.expect(&format!(
"Per thread memory is too small: {}",
per_thread_memory_budget
));
let table_size = compute_table_size(table_num_bits);
let heap_size = per_thread_memory_budget - table_size;
(heap_size, table_num_bits)
@@ -174,13 +179,10 @@ impl<'a> HashMap<'a> {
}
pub fn iter<'b: 'a>(&'b self) -> impl Iterator<Item = (&'a [u8], u32)> + 'b {
self.occupied
.iter()
.cloned()
.map(move |bucket: usize| {
let kv = self.table[bucket];
self.get_key_value(kv.key_value_addr)
})
self.occupied.iter().cloned().map(move |bucket: usize| {
let kv = self.table[bucket];
self.get_key_value(kv.key_value_addr)
})
}
@@ -282,8 +284,10 @@ mod tests {
let s1 = "abcdef";
let s2 = "abcdeg";
for i in 0..5 {
assert_eq!(murmurhash2(&s1[i..5].as_bytes()),
murmurhash2(&s2[i..5].as_bytes()));
assert_eq!(
murmurhash2(&s1[i..5].as_bytes()),
murmurhash2(&s2[i..5].as_bytes())
);
}
}
@@ -303,13 +307,13 @@ mod tests {
let keys: Vec<&'static str> =
vec!["wer qwe qwe qwe ", "werbq weqweqwe2 ", "weraq weqweqwe3 "];
b.iter(|| {
keys.iter()
.map(|&s| s.as_bytes())
.map(murmurhash2::murmurhash2)
.map(|h| h as u64)
.last()
.unwrap()
});
keys.iter()
.map(|&s| s.as_bytes())
.map(murmurhash2::murmurhash2)
.map(|h| h as u64)
.last()
.unwrap()
});
}

40
src/datastruct/stacker/heap.rs
View File

@@ -144,7 +144,8 @@ impl InnerHeap {
addr
} else {
if self.next_heap.is_none() {
info!(r#"Exceeded heap size. The segment will be committed right after indexing this document."#,);
info!(r#"Exceeded heap size. The segment will be committed right
after indexing this document."#,);
self.next_heap = Some(Box::new(InnerHeap::with_capacity(self.buffer_len as usize)));
}
self.next_heap.as_mut().unwrap().allocate_space(num_bytes) + self.buffer_len
@@ -154,10 +155,9 @@ impl InnerHeap {
fn get_slice(&self, bytes_ref: BytesRef) -> &[u8] {
let start = bytes_ref.0;
if start >= self.buffer_len {
self.next_heap
.as_ref()
.unwrap()
.get_slice(BytesRef(start - self.buffer_len))
self.next_heap.as_ref().unwrap().get_slice(BytesRef(
start - self.buffer_len,
))
} else {
let start = start as usize;
let len = NativeEndian::read_u16(&self.buffer[start..start + 2]) as usize;
@@ -167,10 +167,10 @@ impl InnerHeap {
fn get_mut_slice(&mut self, start: u32, stop: u32) -> &mut [u8] {
if start >= self.buffer_len {
self.next_heap
.as_mut()
.unwrap()
.get_mut_slice(start - self.buffer_len, stop - self.buffer_len)
self.next_heap.as_mut().unwrap().get_mut_slice(
start - self.buffer_len,
stop - self.buffer_len,
)
} else {
&mut self.buffer[start as usize..stop as usize]
}
@@ -188,10 +188,9 @@ impl InnerHeap {
fn get_mut(&mut self, addr: u32) -> *mut u8 {
if addr >= self.buffer_len {
self.next_heap
.as_mut()
.unwrap()
.get_mut(addr - self.buffer_len)
self.next_heap.as_mut().unwrap().get_mut(
addr - self.buffer_len,
)
} else {
let addr_isize = addr as isize;
unsafe { self.buffer.as_mut_ptr().offset(addr_isize) }
@@ -200,10 +199,9 @@ impl InnerHeap {
fn get_mut_ref<Item>(&mut self, addr: u32) -> &mut Item {
if addr >= self.buffer_len {
self.next_heap
.as_mut()
.unwrap()
.get_mut_ref(addr - self.buffer_len)
self.next_heap.as_mut().unwrap().get_mut_ref(
addr - self.buffer_len,
)
} else {
let v_ptr_u8 = self.get_mut(addr) as *mut u8;
let v_ptr = v_ptr_u8 as *mut Item;
@@ -213,10 +211,10 @@ impl InnerHeap {
pub fn set<Item>(&mut self, addr: u32, val: &Item) {
if addr >= self.buffer_len {
self.next_heap
.as_mut()
.unwrap()
.set(addr - self.buffer_len, val);
self.next_heap.as_mut().unwrap().set(
addr - self.buffer_len,
val,
);
} else {
let v_ptr: *const Item = val as *const Item;
let v_ptr_u8: *const u8 = v_ptr as *const u8;