rust/tantivy
1
0
Fork 0
mirror of https://github.com/quickwit-oss/tantivy.git synced 2026-05-15 15:50:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

BytesRef is now wrapping an addr

Browse Source
This commit is contained in:
Paul Masurel
2017-06-21 22:32:05 +09:00
parent fb75e60c6e
commit 4ebacb7297
6 changed files with 65 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/datastruct/stacker/hashmap.rs
View File

@@ -148,7 +148,7 @@ impl<'a> HashMap<'a> {
};
}
pub fn iter<'b: 'a>(&'b self) -> impl Iterator<Item=(&'a [u8], u32)> + 'b {
pub fn iter<'b: 'a>(&'b self) -> impl Iterator<Item = (&'a [u8], u32)> + 'b {
let heap: &'a Heap = self.heap;
let table: &'b [KeyValue] = &self.table;
self.occupied
@@ -175,12 +175,12 @@ impl<'a> HashMap<'a> {
let (addr, val): (u32, &mut V) = self.heap.allocate_object();
assert_eq!(addr, key_bytes_ref.addr() + 2 + key_bytes.len() as u32);
self.set_bucket(hash, key_bytes_ref, bucket);
return val
}
if kv.hash == hash {
return val;
} else if kv.hash == hash {
let stored_key: &[u8] = self.get_key(kv.key);
if stored_key == key_bytes {
return self.heap.get_mut_ref(kv.key.addr() + 2 + stored_key.len() as u32);
let expull_addr = kv.key.addr() + 2 + stored_key.len() as u32;
return self.heap.get_mut_ref(expull_addr);
}
}
}

5
src/datastruct/stacker/heap.rs
View File

@@ -166,11 +166,10 @@ impl InnerHeap {
.as_ref()
.unwrap()
.get_slice(BytesRef(start - self.buffer_len))
}
else {
} else {
let start = start as usize;
let len = NativeEndian::read_u16(&self.buffer[start..start + 2]) as usize;
&self.buffer[start + 2.. start + 2 + len]
&self.buffer[start + 2..start + 2 + len]
}
}

69
src/datastruct/stacker/mod.rs
View File

@@ -9,38 +9,39 @@ pub use self::hashmap::HashMap;
// #[test]
// fn test_unrolled_linked_list() {
// let heap = Heap::with_capacity(30_000_000);
// {
// heap.clear();
// let mut ks: Vec<usize> = (1..5).map(|k| k * 100).collect();
// ks.push(2);
// ks.push(3);
// for k in (1..5).map(|k| k * 100) {
// let mut hashmap: HashMap = HashMap::new(10, &heap);
// for j in 0..k {
// for i in 0..500 {
// let mut list: &mut ExpUnrolledLinkedList = hashmap.get_or_create(i.to_string());
// list.push(i * j, &heap);
// }
// }
// for i in 0..500 {
// match hashmap.lookup(i.to_string()) {
// Entry::Occupied(addr) => {
// let v: &mut ExpUnrolledLinkedList = heap.get_mut_ref(addr);
// let mut it = v.iter(addr, &heap);
// for j in 0..k {
// assert_eq!(it.next().unwrap(), i * j);
// }
// assert!(!it.next().is_some());
// }
// _ => {
// panic!("should never happen");
// }
// }
// }
// }
#[test]
fn test_unrolled_linked_list() {
use std::collections;
let heap = Heap::with_capacity(30_000_000);
{
heap.clear();
let mut ks: Vec<usize> = (1..5).map(|k| k * 100).collect();
ks.push(2);
ks.push(3);
for k in (1..5).map(|k| k * 100) {
let mut hashmap: HashMap = HashMap::new(10, &heap);
for j in 0..k {
for i in 0..500 {
let v: &mut ExpUnrolledLinkedList = hashmap.get_or_create(i.to_string());
v.push(i * j, &heap);
}
}
let mut map_addr: collections::HashMap<Vec<u8>, u32> = collections::HashMap::new();
for (key, addr) in hashmap.iter() {
map_addr.insert(Vec::from(key), addr);
}
// }
// }
for i in 0..500 {
let key: String = i.to_string();
let addr: u32 = *map_addr.get(key.as_bytes()).unwrap();
let exp_pull: &ExpUnrolledLinkedList = heap.get_ref(addr);
let mut it = exp_pull.iter(addr, &heap);
for j in 0..k {
assert_eq!(it.next().unwrap(), i * j);
}
assert!(!it.next().is_some());
}
}
}
}

18
src/indexer/index_writer.rs
View File

@@ -250,7 +250,7 @@ fn index_documents(heap: &mut Heap,
segment: Segment,
schema: &Schema,
generation: usize,
document_iterator: &mut Iterator<Item=AddOperations>,
document_iterator: &mut Iterator<Item = AddOperations>,
segment_updater: &mut SegmentUpdater,
mut delete_cursor: DeleteCursor)
-> Result<bool> {
@@ -377,7 +377,8 @@ impl IndexWriter {
loop {
let mut document_iterator = document_receiver_clone.clone().into_iter().peekable();
let mut document_iterator =
document_receiver_clone.clone().into_iter().peekable();
// the peeking here is to avoid
// creating a new segment's files
// if no document are available.
@@ -583,10 +584,11 @@ impl IndexWriter {
pub fn add_document(&mut self, document: Document) -> u64 {
let opstamp = self.stamper.stamp();
let add_operation = AddOperation {
opstamp: opstamp,
opstamp: opstamp,
document: document,
};
self.document_sender.send(AddOperations::from(add_operation));
self.document_sender
.send(AddOperations::from(add_operation));
opstamp
}
@@ -602,13 +604,13 @@ impl IndexWriter {
/// have been added since the creation of the index.
pub fn add_documents(&mut self, documents: Vec<Document>) -> u64 {
let mut ops = Vec::with_capacity(documents.len());
let mut opstamp = 0u64;
let mut opstamp = 0u64;
for doc in documents {
opstamp = self.stamper.stamp();
ops.push(AddOperation {
opstamp: opstamp,
document: doc,
});
opstamp: opstamp,
document: doc,
});
}
self.document_sender.send(AddOperations::from(ops));
opstamp

31
src/indexer/operation.rs
View File

@@ -25,13 +25,9 @@ pub enum AddOperations {
impl AddOperations {
pub fn first_opstamp(&self) -> u64 {
match *self {
AddOperations::Single(ref op) => {
op.opstamp
}
AddOperations::Multiple(ref ops) => {
ops[0].opstamp
}
}
AddOperations::Single(ref op) => op.opstamp,
AddOperations::Multiple(ref ops) => ops[0].opstamp,
}
}
}
@@ -48,19 +44,14 @@ impl From<Vec<AddOperation>> for AddOperations {
}
impl IntoIterator for AddOperations {
type Item = AddOperation;
type IntoIter = Box<Iterator<Item=AddOperation>>;
type IntoIter = Box<Iterator<Item = AddOperation>>;
fn into_iter(self) -> Self::IntoIter {
match self {
AddOperations::Single(op) => {
Box::new(Some(op).into_iter())
}
AddOperations::Multiple(ops) => {
Box::new(ops.into_iter())
}
}
}
}
fn into_iter(self) -> Self::IntoIter {
match self {
AddOperations::Single(op) => Box::new(Some(op).into_iter()),
AddOperations::Multiple(ops) => Box::new(ops.into_iter()),
}
}
}

4
src/termdict/streamdict/streamer.rs
View File

@@ -125,7 +125,7 @@ impl<'a, V> TermStreamerBuilderImpl<'a, V>
origin: origin,
offset_from: 0,
offset_to: data.len(),
current_key: Vec::with_capacity(300)
current_key: Vec::with_capacity(300),
}
}
}
@@ -174,7 +174,7 @@ impl<'a, V> TermStreamer<V> for TermStreamerImpl<'a, V>
self.current_key.truncate(common_length);
let added_length: usize = deserialize_vint(&mut self.cursor) as usize;
self.current_key.extend(&self.cursor[..added_length]);
self.cursor = &self.cursor[added_length..];
self.current_value =
V::deserialize(&mut self.cursor)