From 5815f89a29ad03049a062b8aeb0b29fc47c356b6 Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Mon, 11 Jan 2016 23:10:39 +0900 Subject: [PATCH] compiling again --- Cargo.toml | 2 +- src/core/directory.rs | 120 ++++++++++++++++++++++++++++++++++++------ src/core/global.rs | 4 ++ src/core/mod.rs | 1 + src/core/reader.rs | 15 ++++++ src/main.rs | 4 -- tests/core.rs | 33 +++++++----- 7 files changed, 146 insertions(+), 33 deletions(-) create mode 100644 src/core/reader.rs delete mode 100644 src/main.rs diff --git a/Cargo.toml b/Cargo.toml index 6017171ad..6411054f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "parici" +name = "tantivy" version = "0.1.0" authors = ["Paul Masurel "] diff --git a/src/core/directory.rs b/src/core/directory.rs index 212f7dc76..53607e0aa 100644 --- a/src/core/directory.rs +++ b/src/core/directory.rs @@ -1,8 +1,8 @@ - extern crate memmap; use self::memmap::{Mmap, Protection}; use std::path::PathBuf; +use std::collections::HashMap; use std::fs::File; use std::io; use std::rc::Rc; @@ -11,7 +11,7 @@ use std::rc::Rc; pub struct SegmentId(String); pub trait Dir { - fn get_file(&self, segment_id: &SegmentId, component: SegmentComponent) -> Result; + fn get_file<'a>(&'a self, segment_id: &SegmentId, component: SegmentComponent) -> Result<&'a MemoryPointer, io::Error>; // { } #[derive(Clone)] @@ -27,16 +27,24 @@ impl Directory { } } + fn from(directory: T) -> Directory { + Directory { + dir: Rc::new(directory), + } + } + pub fn open(path_str: &str) -> Directory { let path = PathBuf::from(path_str); - Directory { - dir: Rc::new(FileDirectory::for_path(path)), - } + Directory::from(FileDirectory::for_path(path)) + } + + pub fn in_mem() -> Directory { + Directory::from(MemDirectory::new()) } } impl Dir for Directory { - fn get_file(&self, segment_id: &SegmentId, component: SegmentComponent) -> Result { + fn get_file<'a>(&'a self, segment_id: &SegmentId, component: SegmentComponent) -> Result<&'a MemoryPointer, io::Error> { self.dir.get_file(segment_id, component) } } @@ -52,7 +60,6 @@ pub struct Segment { } impl Segment { - fn path_suffix(component: SegmentComponent)-> &'static str { match component { SegmentComponent::POSTINGS => ".pstgs", @@ -60,17 +67,19 @@ impl Segment { } } - fn get_file(&self, component: SegmentComponent) -> Result { + fn get_file<'a>(&'a self, component: SegmentComponent) -> Result<&'a MemoryPointer, io::Error> { self.directory.get_file(&self.segment_id, component) } - - pub fn open(&self, component: SegmentComponent) -> Result { - let file = try!(self.get_file(component)); - Mmap::open(&file, Protection::Read) - } } + + + + +////////////////////////////////////////////////////////// +// FileDirectory + pub struct FileDirectory { index_path: PathBuf, } @@ -83,12 +92,93 @@ impl FileDirectory { } } + + + +///////////////////////////////////////////////////////// +// MemoryPointer + +pub trait MemoryPointer { + fn len(&self) -> usize; + fn ptr(&self) -> *const u8; +} + +///////////////////////////////////////////////////////// +// ResidentMemoryPointer + +pub struct ResidentMemoryPointer { + data: Box<[u8]>, + len: usize, +} + +impl MemoryPointer for ResidentMemoryPointer { + fn len(&self) -> usize { + self.len + } + fn ptr(&self) -> *const u8 { + &self.data[0] + } +} + + +///////////////////////////////////////////////////////// +// +// + + +struct MmapMemory(Mmap); + +impl MemoryPointer for MmapMemory { + fn len(&self) -> usize { + let &MmapMemory(ref mmap) = self; + mmap.len() + } + fn ptr(&self) -> *const u8 { + let &MmapMemory(ref mmap) = self; + mmap.ptr() + } +} + impl Dir for FileDirectory { - fn get_file(&self, segment_id: &SegmentId, component: SegmentComponent) -> Result { + fn get_file<'a>(&'a self, segment_id: &SegmentId, component: SegmentComponent) -> Result<&'a MemoryPointer, io::Error> { let mut res = self.index_path.clone(); let SegmentId(ref segment_id_str) = *segment_id; let filename = String::new() + segment_id_str + "." + Segment::path_suffix(component); res.push(filename); - File::open(res) + let file = try!(File::open(res)); + let mmap = MmapMemory(try!(Mmap::open(&file, Protection::Read))); + // let boxed_mmap: Box = Box::new(mmap); + // Ok(boxed_mmap) + Err(io::Error::new(io::ErrorKind::AddrInUse, "eee")) + } +} + +////////////////////////////////////////////////////////// +// FileDirectory +// + +pub struct MemDirectory { + dir: HashMap>, +} + +impl MemDirectory { + pub fn new()-> MemDirectory { + MemDirectory { + dir: HashMap::new(), + } + } +} + +impl Dir for MemDirectory { + fn get_file<'a>(&'a self, segment_id: &SegmentId, component: SegmentComponent) -> Result<&'a MemoryPointer, io::Error> { + let SegmentId(ref segment_id_str) = *segment_id; + let mut path = PathBuf::from(segment_id_str); + path.push(Segment::path_suffix(component)); + match self.dir.get(&path) { + Some(buf) => { + Ok(buf.as_ref()) + }, + None => Err(io::Error::new(io::ErrorKind::NotFound, "File does not exists")), + } } } diff --git a/src/core/global.rs b/src/core/global.rs index a64f95b70..c3f986d63 100644 --- a/src/core/global.rs +++ b/src/core/global.rs @@ -1,2 +1,6 @@ pub type DocId = usize; + +// pub trait SeekableIterator: Iterator { +// pub fn seek(&mut self, el: &T) -> bool; +// } diff --git a/src/core/mod.rs b/src/core/mod.rs index ff5784af8..97729423e 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -5,5 +5,6 @@ pub mod schema; pub mod directory; pub mod writer; pub mod analyzer; +pub mod reader; pub use core::global::DocId; diff --git a/src/core/reader.rs b/src/core/reader.rs new file mode 100644 index 000000000..1779bfb53 --- /dev/null +++ b/src/core/reader.rs @@ -0,0 +1,15 @@ +use core::directory::Directory; + +pub struct IndexReader { + directory: Directory, +} + +impl IndexReader { + + pub fn open(directory: &Directory) -> IndexReader { + IndexReader { + directory: (*directory).clone(), + } + } + +} diff --git a/src/main.rs b/src/main.rs deleted file mode 100644 index 7db121b6a..000000000 --- a/src/main.rs +++ /dev/null @@ -1,4 +0,0 @@ -extern crate parici; - -fn main() { -} diff --git a/tests/core.rs b/tests/core.rs index 0a481bab5..6e16a0ace 100644 --- a/tests/core.rs +++ b/tests/core.rs @@ -1,13 +1,14 @@ -extern crate parici; +extern crate tantivy; extern crate itertools; -use parici::core::DocId; -use parici::core::postings::{VecPostings, intersection}; -use parici::core::postings::Postings; -use parici::core::analyzer::tokenize; -use parici::core::writer::IndexWriter; -use parici::core::directory::Directory; -use parici::core::schema::{Field, Document}; +use tantivy::core::DocId; +use tantivy::core::postings::{VecPostings, intersection}; +use tantivy::core::postings::Postings; +use tantivy::core::analyzer::tokenize; +use tantivy::core::writer::IndexWriter; +use tantivy::core::directory::Directory; +use tantivy::core::schema::{Field, Document}; +use tantivy::core::reader::IndexReader; #[test] fn test_intersection() { @@ -26,9 +27,15 @@ fn test_tokenizer() { #[test] fn test_indexing() { - let directory = Directory::open("toto"); - let mut index_writer = IndexWriter::open(&directory); - let mut doc = Document::new(); - doc.set(Field("text"), &String::from("toto")); - index_writer.add(doc); + let directory = Directory::in_mem(); + { + let mut index_writer = IndexWriter::open(&directory); + let mut doc = Document::new(); + doc.set(Field("text"), "toto"); + index_writer.add(doc); + index_writer.sync().unwrap(); + } + { + let index_reader = IndexReader::open(&directory); + } }