From e244bd84037dea31d477cc9cccf468a5f945d753 Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Mon, 18 Jan 2016 09:01:16 +0900 Subject: [PATCH] blip --- Cargo.toml | 1 + src/core/directory.rs | 313 ++++++------------------------------------ 2 files changed, 45 insertions(+), 269 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index b5c31321b..c62bd8275 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,3 +11,4 @@ lazy_static = "0.1.*" regex = "0.1" fst = "0.1.26" rand = "0.3.13" +atomicwrites = "0.0.14" diff --git a/src/core/directory.rs b/src/core/directory.rs index c874f0561..71a873aaa 100644 --- a/src/core/directory.rs +++ b/src/core/directory.rs @@ -3,6 +3,7 @@ extern crate memmap; use self::memmap::{Mmap, Protection}; use std::path::PathBuf; use std::collections::HashMap; +use std::collections::hash_map::Entry; use std::fs::File; use std::io::Write; use std::io::BufWriter; @@ -10,6 +11,7 @@ use std::io; use std::borrow::Borrow; use std::borrow::BorrowMut; use std::rc::Rc; +use std::fmt; use std::ops::Deref; use std::cell::RefCell; use std::sync::Arc; @@ -28,18 +30,35 @@ pub fn generate_segment_name() -> SegmentId { SegmentId( String::from("_") + &random_name) } - -#[derive(Clone, Debug)] +#[derive(Clone)] pub struct Directory { index_path: PathBuf, - // mmap_cache: RefCell>, + mmap_cache: RefCell>, +} + +impl fmt::Debug for Directory { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "Directory({:?})", self.index_path) + } +} + +fn open_mmap(full_path: &PathBuf) -> Result { + match Mmap::open_path(full_path.clone(), Protection::Read) { + Ok(mmapped_file) => Ok(SharedMmapMemory::new(mmapped_file)), + Err(ioerr) => { + // TODO add file + let error_msg = format!("Read-Only MMap of {:?} failed", full_path); + return Err(Error::IOError(ioerr.kind(), error_msg)); + } + } } impl Directory { pub fn from(filepath: &str) -> Directory { Directory { - index_path: PathBuf::from(filepath) + index_path: PathBuf::from(filepath), + mmap_cache: RefCell::new(HashMap::new()), } } @@ -69,6 +88,15 @@ impl Directory { } } } + + fn open_readable<'a>(&self, relative_path: &PathBuf) -> Result { + let full_path = self.resolve_path(relative_path); + let mut cache = self.mmap_cache.borrow_mut(); + if !cache.contains_key(&full_path) { + cache.insert(full_path.clone(), try!(open_mmap(&full_path)) ); + } + Ok(cache.get(&full_path).unwrap().clone()) + } } ///////////////////////// @@ -101,275 +129,22 @@ impl<'a> Segment<'a> { PathBuf::from(filename) } - // pub fn get_data(&self, component: SegmentComponent) -> Result>> { - // self.directory.get_data(&self.segment_id, component) - // } - + pub fn get_data(&self, component: SegmentComponent) -> Result { + let path = self.get_relative_path(component); + self.directory.open_readable(&path) + } + pub fn open_writable(&self, component: SegmentComponent) -> Result { let path = self.get_relative_path(component); self.directory.open_writable(&path) } } +#[derive(Clone)] +pub struct SharedMmapMemory(Arc); - - -// #[derive(Clone)] -// pub struct Directory { -// dir: Rc, -// } -// -// impl Directory { -// fn segment(&self, segment_id: &SegmentId) -> Segment { -// Segment { -// directory: self.dir.clone(), -// segment_id: segment_id.clone() -// } -// } -// -// pub fn new_segment(&self,) -> Segment { -// self.segment(&generate_segment_name()) -// } -// -// fn from(directory: T) -> Directory { -// Directory { -// dir: Rc::new(directory), -// } -// } -// -// // pub fn open(path_str: &str) -> Directory { -// // let path = PathBuf::from(path_str); -// // Directory::from(FileDirectory::for_path(path)) -// // } -// -// pub fn in_mem() -> Directory { -// Directory::from(MemDirectory::new()) -// } -// } -// -// impl Dir for Directory { -// // fn get_data(&self, segment_id: &SegmentId, component: SegmentComponent) -> Result>> { -// // self.dir.get_data(segment_id, component) -// // } -// - - -// fn open_writable<'a>(&'a self, path: &PathBuf) -> Result>> { -// self.dir.open_writable(path) -// } -// } - -// pub enum SegmentComponent { -// POSTINGS, -// POSITIONS, -// TERMS, -// } -// -// pub struct Segment { -// directory: Rc, -// segment_id: SegmentId, -// } -// -// impl Segment { -// fn path_suffix(component: SegmentComponent)-> &'static str { -// match component { -// SegmentComponent::POSTINGS => ".idx", -// SegmentComponent::POSITIONS => ".pos", -// SegmentComponent::TERMS => ".term", -// } -// } -// -// fn get_path(&self, component: SegmentComponent) -> PathBuf { -// let mut filepath = self.index_path.clone(); -// let SegmentId(ref segment_id_str) = self.segment_id; -// let filename = String::new() + segment_id_str + "." + Segment::path_suffix(component); -// filepath.push(filename); -// filepath -// } -// -// // pub fn get_data(&self, component: SegmentComponent) -> Result>> { -// // self.directory.get_data(&self.segment_id, component) -// // } -// pub fn open_writable(&self, component: SegmentComponent) -> Result>> { -// let path = self.get_path(component); -// self.directory.open_writable(path) -// } -// } - - - -// -// ///////////////////////////////////////////////////////// -// // ResidentMemoryPointer -// -// pub struct ResidentMemoryPointer { -// data: Box<[u8]>, -// } -// -// impl Borrow<[u8]> for ResidentMemoryPointer { -// fn borrow(&self) -> &[u8] { -// self.data.deref() -// } -// } -// -// -// ////////////////////////////////////////////////////////// -// // MemDirectory -// // -// pub struct MemDirectory { -// dir: HashMap>, -// } -// -// impl MemDirectory { -// pub fn new()-> MemDirectory { -// MemDirectory { -// dir: HashMap::new(), -// } -// } -// -// // fn get_path(&self, segment_id: &SegmentId, component: SegmentComponent) -> PathBuf { -// // let mut filepath = self.index_path.clone(); -// // let SegmentId(ref segment_id_str) = *segment_id; -// // let filename = String::new() + segment_id_str + "." + Segment::path_suffix(component); -// // filepath.push(filename); -// // filepath -// // } -// } -// -// impl Directory for MemDirectory { -// -// type Write = Rc; -// -// // fn get_data(&self, segment_id: &SegmentId, component: SegmentComponent) -> Result>> { -// // let path = self.get_path(segment_id, component); -// // match self.dir.get(&path) { -// // Some(buf) => Ok(buf.clone()), -// // None => Err(Error::FileNotFound(String::from("File not found"))), // TODO add filename -// // } -// // } -// -// fn open_writable<'a>(&'a self, path: &PathBuf) -> Result> { -// if self.dir.contains_key(path) { -// return Err(Error::ReadOnly(String::from("Cannot open an already written buffer."))); -// } -// self.dir.insert(path.clone(), Rc::new(String::new())); -// self.dir.get(path); -// Ok(Box::new()) -// -// } -// } -// -// -// -// -// -// -// -// -// -// -// -// -// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -// -// -// #[derive(Clone)] -// pub struct SharedMmapMemory(Arc); -// -// impl SharedMmapMemory { -// pub fn new(mmap_memory: MmapMemory) -> SharedMmapMemory { -// SharedMmapMemory(Arc::new(mmap_memory)) -// } -// } -// -// -// ///////////////////////////////////////////////////////// -// // MmapMemory -// // -// -// pub struct MmapMemory(Mmap); -// -// impl MemoryPointer for MmapMemory { -// fn data(&self) -> &[u8] { -// let &MmapMemory(ref mmap) = self; -// unsafe { -// mmap.as_slice() -// } -// } -// } -// -// ////////////////////////////////////////////////////////// -// // FileDirectory -// -// pub struct FileDirectory { -// index_path: PathBuf, -// mmap_cache: RefCell>, -// } -// -// impl FileDirectory { -// pub fn for_path(path: PathBuf)-> FileDirectory { -// FileDirectory { -// index_path: path, -// mmap_cache: RefCell::new(HashMap::new()), -// } -// } -// -// fn get_or_open_mmap(&self, filepath: &PathBuf)->Result<[u8]> { -// if !self.mmap_cache.borrow().contains_key(filepath) { -// let file = try!(File::open(filepath)); -// let mmap = MmapMemory(try!(Mmap::open(&file, Protection::Read))); -// self.mmap_cache.borrow_mut().insert(filepath.clone(), SharedMmapMemory::new(mmap)); -// } -// let shared_map: SharedMmapMemory = self.mmap_cache.borrow().get(filepath).unwrap().clone(); -// Ok(shared_map) -// } -// -// fn get_path(&self, segment_id: &SegmentId, component: SegmentComponent) -> PathBuf { -// let mut filepath = self.index_path.clone(); -// let SegmentId(ref segment_id_str) = *segment_id; -// let filename = String::new() + segment_id_str + "." + Segment::path_suffix(component); -// filepath.push(filename); -// filepath -// } -// } -// -// impl Dir for FileDirectory { -// fn get_data(&self, segment_id: &SegmentId, component: SegmentComponent) -> Result<[u8]> { -// let filepath = self.get_path(segment_id, component); -// self.get_or_open_mmap(&filepath) -// } -// -// fn open_writable<'a>(&'a self, segment_id: &SegmentId, component: SegmentComponent) -> Result { -// Err(Error::IOError("e")) -// } -// } +impl SharedMmapMemory { + pub fn new(mmap_memory: Mmap) -> SharedMmapMemory { + SharedMmapMemory(Arc::new(mmap_memory)) + } +}