rust/tantivy
1
0
Fork 0
mirror of https://github.com/quickwit-oss/tantivy.git synced 2026-01-06 01:02:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

blip

Browse Source
This commit is contained in:
Paul Masurel
2016-01-18 09:01:16 +09:00
parent c054b7410a
commit e244bd8403
2 changed files with 45 additions and 269 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.toml
View File

@@ -11,3 +11,4 @@ lazy_static = "0.1.*"
regex = "0.1"
fst = "0.1.26"
rand = "0.3.13"
atomicwrites = "0.0.14"

313
src/core/directory.rs
View File

@@ -3,6 +3,7 @@ extern crate memmap;
use self::memmap::{Mmap, Protection};
use std::path::PathBuf;
use std::collections::HashMap;
use std::collections::hash_map::Entry;
use std::fs::File;
use std::io::Write;
use std::io::BufWriter;
@@ -10,6 +11,7 @@ use std::io;
use std::borrow::Borrow;
use std::borrow::BorrowMut;
use std::rc::Rc;
use std::fmt;
use std::ops::Deref;
use std::cell::RefCell;
use std::sync::Arc;
@@ -28,18 +30,35 @@ pub fn generate_segment_name() -> SegmentId {
SegmentId( String::from("_") + &random_name)
}
#[derive(Clone, Debug)]
#[derive(Clone)]
pub struct Directory {
index_path: PathBuf,
// mmap_cache: RefCell<HashMap<PathBuf, SharedMmapMemory >>,
mmap_cache: RefCell<HashMap<PathBuf, SharedMmapMemory >>,
}
impl fmt::Debug for Directory {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "Directory({:?})", self.index_path)
}
}
fn open_mmap(full_path: &PathBuf) -> Result<SharedMmapMemory> {
match Mmap::open_path(full_path.clone(), Protection::Read) {
Ok(mmapped_file) => Ok(SharedMmapMemory::new(mmapped_file)),
Err(ioerr) => {
// TODO add file
let error_msg = format!("Read-Only MMap of {:?} failed", full_path);
return Err(Error::IOError(ioerr.kind(), error_msg));
}
}
}
impl Directory {
pub fn from(filepath: &str) -> Directory {
Directory {
index_path: PathBuf::from(filepath)
index_path: PathBuf::from(filepath),
mmap_cache: RefCell::new(HashMap::new()),
}
}
@@ -69,6 +88,15 @@ impl Directory {
}
}
}
fn open_readable<'a>(&self, relative_path: &PathBuf) -> Result<SharedMmapMemory> {
let full_path = self.resolve_path(relative_path);
let mut cache = self.mmap_cache.borrow_mut();
if !cache.contains_key(&full_path) {
cache.insert(full_path.clone(), try!(open_mmap(&full_path)) );
}
Ok(cache.get(&full_path).unwrap().clone())
}
}
/////////////////////////
@@ -101,275 +129,22 @@ impl<'a> Segment<'a> {
PathBuf::from(filename)
}
// pub fn get_data(&self, component: SegmentComponent) -> Result<Box<Borrow<[u8]>>> {
// self.directory.get_data(&self.segment_id, component)
// }
pub fn get_data(&self, component: SegmentComponent) -> Result<SharedMmapMemory> {
let path = self.get_relative_path(component);
self.directory.open_readable(&path)
}
pub fn open_writable(&self, component: SegmentComponent) -> Result<File> {
let path = self.get_relative_path(component);
self.directory.open_writable(&path)
}
}
#[derive(Clone)]
pub struct SharedMmapMemory(Arc<Mmap>);
// #[derive(Clone)]
// pub struct Directory {
// dir: Rc<Dir>,
// }
//
// impl Directory {
// fn segment(&self, segment_id: &SegmentId) -> Segment {
// Segment {
// directory: self.dir.clone(),
// segment_id: segment_id.clone()
// }
// }
//
// pub fn new_segment(&self,) -> Segment {
// self.segment(&generate_segment_name())
// }
//
// fn from<T: Dir + 'static>(directory: T) -> Directory {
// Directory {
// dir: Rc::new(directory),
// }
// }
//
// // pub fn open(path_str: &str) -> Directory {
// // let path = PathBuf::from(path_str);
// // Directory::from(FileDirectory::for_path(path))
// // }
//
// pub fn in_mem() -> Directory {
// Directory::from(MemDirectory::new())
// }
// }
//
// impl Dir for Directory {
// // fn get_data(&self, segment_id: &SegmentId, component: SegmentComponent) -> Result<Box<Borrow<[u8]>>> {
// // self.dir.get_data(segment_id, component)
// // }
//
// fn open_writable<'a>(&'a self, path: &PathBuf) -> Result<Box<BorrowMut<Write>>> {
// self.dir.open_writable(path)
// }
// }
// pub enum SegmentComponent {
// POSTINGS,
// POSITIONS,
// TERMS,
// }
//
// pub struct Segment {
// directory: Rc<Dir>,
// segment_id: SegmentId,
// }
//
// impl Segment {
// fn path_suffix(component: SegmentComponent)-> &'static str {
// match component {
// SegmentComponent::POSTINGS => ".idx",
// SegmentComponent::POSITIONS => ".pos",
// SegmentComponent::TERMS => ".term",
// }
// }
//
// fn get_path(&self, component: SegmentComponent) -> PathBuf {
// let mut filepath = self.index_path.clone();
// let SegmentId(ref segment_id_str) = self.segment_id;
// let filename = String::new() + segment_id_str + "." + Segment::path_suffix(component);
// filepath.push(filename);
// filepath
// }
//
// // pub fn get_data(&self, component: SegmentComponent) -> Result<Box<Borrow<[u8]>>> {
// // self.directory.get_data(&self.segment_id, component)
// // }
// pub fn open_writable(&self, component: SegmentComponent) -> Result<Box<BorrowMut<Write>>> {
// let path = self.get_path(component);
// self.directory.open_writable(path)
// }
// }
//
// /////////////////////////////////////////////////////////
// // ResidentMemoryPointer
//
// pub struct ResidentMemoryPointer {
// data: Box<[u8]>,
// }
//
// impl Borrow<[u8]> for ResidentMemoryPointer {
// fn borrow(&self) -> &[u8] {
// self.data.deref()
// }
// }
//
//
// //////////////////////////////////////////////////////////
// // MemDirectory
// //
// pub struct MemDirectory {
// dir: HashMap<PathBuf, Rc<String>>,
// }
//
// impl MemDirectory {
// pub fn new()-> MemDirectory {
// MemDirectory {
// dir: HashMap::new(),
// }
// }
//
// // fn get_path(&self, segment_id: &SegmentId, component: SegmentComponent) -> PathBuf {
// // let mut filepath = self.index_path.clone();
// // let SegmentId(ref segment_id_str) = *segment_id;
// // let filename = String::new() + segment_id_str + "." + Segment::path_suffix(component);
// // filepath.push(filename);
// // filepath
// // }
// }
//
// impl Directory for MemDirectory {
//
// type Write = Rc<String>;
//
// // fn get_data(&self, segment_id: &SegmentId, component: SegmentComponent) -> Result<Box<Borrow<[u8]>>> {
// // let path = self.get_path(segment_id, component);
// // match self.dir.get(&path) {
// // Some(buf) => Ok(buf.clone()),
// // None => Err(Error::FileNotFound(String::from("File not found"))), // TODO add filename
// // }
// // }
//
// fn open_writable<'a>(&'a self, path: &PathBuf) -> Result<Rc<String>> {
// if self.dir.contains_key(path) {
// return Err(Error::ReadOnly(String::from("Cannot open an already written buffer.")));
// }
// self.dir.insert(path.clone(), Rc::new(String::new()));
// self.dir.get(path);
// Ok(Box::new())
//
// }
// }
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
// #[derive(Clone)]
// pub struct SharedMmapMemory(Arc<MmapMemory>);
//
// impl SharedMmapMemory {
// pub fn new(mmap_memory: MmapMemory) -> SharedMmapMemory {
// SharedMmapMemory(Arc::new(mmap_memory))
// }
// }
//
//
// /////////////////////////////////////////////////////////
// // MmapMemory
// //
//
// pub struct MmapMemory(Mmap);
//
// impl MemoryPointer for MmapMemory {
// fn data(&self) -> &[u8] {
// let &MmapMemory(ref mmap) = self;
// unsafe {
// mmap.as_slice()
// }
// }
// }
//
// //////////////////////////////////////////////////////////
// // FileDirectory
//
// pub struct FileDirectory {
// index_path: PathBuf,
// mmap_cache: RefCell<HashMap<PathBuf, SharedMmapMemory >>,
// }
//
// impl FileDirectory {
// pub fn for_path(path: PathBuf)-> FileDirectory {
// FileDirectory {
// index_path: path,
// mmap_cache: RefCell::new(HashMap::new()),
// }
// }
//
// fn get_or_open_mmap(&self, filepath: &PathBuf)->Result<[u8]> {
// if !self.mmap_cache.borrow().contains_key(filepath) {
// let file = try!(File::open(filepath));
// let mmap = MmapMemory(try!(Mmap::open(&file, Protection::Read)));
// self.mmap_cache.borrow_mut().insert(filepath.clone(), SharedMmapMemory::new(mmap));
// }
// let shared_map: SharedMmapMemory = self.mmap_cache.borrow().get(filepath).unwrap().clone();
// Ok(shared_map)
// }
//
// fn get_path(&self, segment_id: &SegmentId, component: SegmentComponent) -> PathBuf {
// let mut filepath = self.index_path.clone();
// let SegmentId(ref segment_id_str) = *segment_id;
// let filename = String::new() + segment_id_str + "." + Segment::path_suffix(component);
// filepath.push(filename);
// filepath
// }
// }
//
// impl Dir for FileDirectory {
// fn get_data(&self, segment_id: &SegmentId, component: SegmentComponent) -> Result<[u8]> {
// let filepath = self.get_path(segment_id, component);
// self.get_or_open_mmap(&filepath)
// }
//
// fn open_writable<'a>(&'a self, segment_id: &SegmentId, component: SegmentComponent) -> Result<Write + 'a> {
// Err(Error::IOError("e"))
// }
// }
impl SharedMmapMemory {
pub fn new(mmap_memory: Mmap) -> SharedMmapMemory {
SharedMmapMemory(Arc::new(mmap_memory))
}
}